bump version to 1.29.0

Return BTreeMap instead of HashMap from get_info()

.circleci/config.yml

@@ -36,12 +36,6 @@ jobs:
     executor: default
     steps:
       - checkout
-      - run:
-          name: Update submodules
-          command: git submodule update --init --recursive
-      - run:
-          name: Calculate dependencies
-          command: cargo generate-lockfile
       - restore_cache:
           keys:
             - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
@@ -49,7 +43,6 @@ jobs:
       - run: rustup default $(cat rust-toolchain)
       - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
       - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
-      - run: cargo update
       - run: cargo fetch
       - run: rustc +stable --version
       - run: rustc +$(cat rust-toolchain) --version
@@ -91,7 +84,6 @@ jobs:
             curl https://sh.rustup.rs -sSf | sh -s -- -y
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
-      - run: cargo update
       - run: cargo fetch
       - run:
           name: Test
@@ -128,32 +120,19 @@ jobs:
           paths:
             - c-docs
 
-  # build_test_docs_wheel:
-  #   docker:
-  #     - image: deltachat/coredeps
-  #       environment:
-  #           TESTS: 1
-  #           DOCS: 1
-  #   working_directory: /mnt/crate
-  #   steps:
-  #     - *restore-workspace
-  #     - *restore-cache
-  #     - run:
-  #         name: build docs, run tests and build wheels
-  #         command: ci_scripts/run-python.sh
-  #     - run:
-  #          name: copying docs and wheels to workspace
-  #          command: |
-  #            mkdir -p workspace/python
-  #            # cp -av docs workspace/c-docs
-  #            cp -av python/.docker-tox/wheelhouse workspace/
-  #            cp -av python/doc/_build/ workspace/py-docs
-  #     - persist_to_workspace:
-  #         root: workspace
-  #         paths:
-  #           # - c-docs
-  #           - py-docs
-  #           - wheelhouse
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash ci_scripts/remote_python_packaging.sh 
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
 
   remote_tests_rust:
     machine: true
@@ -165,10 +144,7 @@ jobs:
     machine: true
     steps:
       - checkout
-      #- attach_workspace:
-      #    at: workspace
       - run: ci_scripts/remote_tests_python.sh
-    # workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   upload_docs_wheels:
     machine: true
@@ -176,6 +152,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
+      - run: pyenv versions
       - run: pyenv global 3.5.2
       - run: ls -laR workspace
       - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
@@ -187,7 +164,7 @@ jobs:
       - *restore-cache
       - run:
           name: Run cargo clippy
-          command: cargo clippy --all
+          command: cargo clippy 
 
 
 workflows:
@@ -195,24 +172,44 @@ workflows:
 
   test:
     jobs:
-      - cargo_fetch
+      # - cargo_fetch
 
-      - remote_tests_rust
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
 
-      - remote_tests_python
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
 
-      # - upload_docs_wheels:
+      - remote_python_packaging:
+          requires: 
+             - remote_tests_python
+             - remote_tests_rust 
+          filters:
+            tags:
+              only: /.*/
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            tags:
+              only: /.*/
+      # - rustfmt:
       #     requires:
-      #       - build_test_docs_wheel
-      #       - build_doxygen
-      - rustfmt:
-          requires:
-            - cargo_fetch
-      - clippy:
-          requires:
-           - cargo_fetch
+      #       - cargo_fetch
+      # - clippy:
+      #     requires:
+      #      - cargo_fetch
 
-      - build_doxygen
+      - build_doxygen:
+          filters:
+            tags:
+              only: /.*/
 
       # Linux Desktop 64bit
       # - test_x86_64-unknown-linux-gnu:

.github/workflows/code-quality.yml

@@ -0,0 +1,48 @@
+on: push
+name: Code Quality
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-12
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: --workspace --examples --tests --all-features
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-12
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: nightly-2020-03-12
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features

CHANGELOG.md

@@ -1,5 +1,317 @@
 # Changelog 
 
+## 1.29.0
+
+- new config options `delete_device_after` and `delete_server_after`,
+  each taking an amount of seconds after which messages
+  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
+
+- new api `dc_estimate_deletion_cnt()` to estimate the effect
+  of `delete_device_after` and `delete_server_after`
+
+- use Ed25519 keys by default, these keys are much shorter
+  than RSA keys, which results in saving traffic and speed improvements #1362
+
+- improve message ellipsizing #1397 #1430
+
+- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
+
+- do not show badly formatted non-delta-messages as empty #1384
+
+- try over SMTP on potentially recoverable error 5.5.0 #1379
+
+- remove device-chat from forward-to-chat-list #1367
+
+- improve group-handling #1368
+
+- `dc_get_info()` returns uptime (how long the context is in use)
+
+- python improvements and adaptions #1408 #1415
+
+- log to the stdout and stderr in tests #1416
+
+- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
+
+- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
+
+- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
+
+
+## 1.28.0
+
+- new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()
+  that will sort the "saved messages" chat to the top of the chatlist #1336
+- mark mails as being deleted from server in dc_empty_server() #1333
+- fix interaction with servers that do not allow folder creation on root-level;
+  use path separator as defined by the email server #1359
+- fix group creation if group was created by non-delta clients #1357
+- fix showing replies from non-delta clients #1353
+- fix member list on rejoining left groups #1343
+- fix crash when using empty groups #1354
+- fix potential crash on special names #1350
+
+
+## 1.27.0
+
+- handle keys reliably on armv7 #1327
+
+
+## 1.26.0
+
+- change generated key type back to RSA as shipped versions
+  have problems to encrypt to Ed25519 keys
+
+- update rPGP to encrypt reliably to Ed25519 keys;
+  one of the next versions can finally use Ed25519 keys then
+
+
+## 1.25.0
+
+- save traffic by downloading only messages that are really displayed #1236
+
+- change generated key type to Ed25519, these keys are much shorter
+  than RSA keys, which results in saving traffic and speed improvements #1287
+
+- improve key handling #1237 #1240 #1242 #1247
+
+- mute handling, apis are dc_set_chat_mute_duration()
+  dc_chat_is_muted() and dc_chat_get_remaining_mute_duration() #1143
+
+- pinning chats, new apis are dc_set_chat_visibility() and
+  dc_chat_get_visibility() #1248
+
+- add dc_provider_new_from_email() api that queries the new, integrated
+  provider-database #1207
+
+- account creation by scanning a qr code
+  in the DCACCOUNT scheme (https://mailadm.readthedocs.io),
+  new api is dc_set_config_from_qr() #1249
+
+- if possible, dc_join_securejoin(), returns the new chat-id immediately
+  and does the handshake in background #1225
+
+- update imap and smtp dependencies #1115
+
+- check for MOVE capability before using MOVE command #1263
+
+- allow inline attachments from RFC 2183 #1280
+
+- fix updating names from incoming mails #1298
+
+- fix error messages shown on import #1234
+
+- directly attempt to re-connect if the smtp connection is maybe stale #1296
+
+- improve adding group members #1291
+
+- improve rust-api #1261
+
+- cleanup #1302 #1283 #1282 #1276 #1270-#1274 #1267 #1258-#1260
+  #1257 #1239 #1231 #1224
+
+- update spec #1286 #1291
+
+
+## 1.0.0-beta.24
+
+- fix oauth2/gmail bug introduced in beta23 (not used in releases) #1219
+
+- fix panic when receiving eg. cyrillic filenames #1216
+
+- delete all consumed secure-join handshake messagess #1209 #1212
+
+- rust-level cleanups #1218 #1217 #1210 #1205
+
+- python-level cleanups #1204 #1202 #1201
+
+
+## 1.0.0-beta.23
+
+- #1197 fix imap-deletion of messages 
+
+- #1171 Combine multiple MDNs into a single mail, reducing traffic 
+
+- #1155 fix to not send out gossip always, reducing traffic
+
+- #1160 fix reply-to-encrypted determination 
+
+- #1182 Add "Auto-Submitted: auto-replied" header to MDNs
+
+- #1194 produce python wheels again, fix c/py.delta.chat
+  master-deployment 
+
+- rust-level housekeeping and improvements #1161 #1186 #1185 #1190 #1194 #1199 #1191 #1190 #1184 and more
+
+- #1063 clarify licensing 
+
+- #1147 use mailparse 0.10.2 
+
+
+## 1.0.0-beta.22
+
+- #1095 normalize email lineends to CRLF
+
+- #1095 enable link-time-optimization, saves eg. on android 11 mb
+
+- #1099 fix import regarding devicechats
+
+- #1092 improve logging
+
+- #1096 #1097 #1094 #1090 #1091 internal cleanups
+
+## 1.0.0-beta.21
+
+- #1078 #1082 ensure RFC compliance by producing 78 column lines for
+  encoded attachments. 
+
+- #1080 don't recreate and thus break group membership if an unknown 
+  sender (or mailer-daemon) sends a message referencing the group chat 
+
+- #1081 #1079 some internal cleanups 
+
+- update imap-proto dependency, to fix yandex/oauth 
+
+## 1.0.0-beta.20
+
+- #1074 fix OAUTH2/gmail
+- #1072 fix group members not appearing in contact list
+- #1071 never block interrupt_idle (thus hopefully also not on maybe_network())
+- #1069 reduce smtp-timeout to 30 seconds
+- #1066 #1065 avoid unwrap in dehtml, make literals more readable
+
+## 1.0.0-beta.19
+
+- #1058 timeout smtp-send if it doesn't complete in 15 minutes 
+
+- #1059 trim down logging
+
+## 1.0.0-beta.18
+
+- #1056 avoid panicking when we couldn't read imap-server's greeting
+  message 
+
+- #1055 avoid panicking when we don't have a selected folder
+
+- #1052 #1049 #1051 improve logging to add thread-id/name and
+  file/lineno to each info/warn message.
+
+- #1050 allow python bindings to initialize Account with "os_name".
+
+
+## 1.0.0-beta.17
+
+- #1044 implement avatar recoding to 192x192 in core to keep file sizes small. 
+
+- #1024 fix #1021 SQL/injection malformed Chat-Group-Name breakage
+
+- #1036 fix smtp crash by pulling in a fixed async-smtp 
+
+- #1039 fix read-receipts appearing as normal messages when you change
+  MDN settings 
+
+- #1040 do not panic on SystemTimeDifference
+
+- #1043 avoid potential crashes in malformed From/Chat-Disposition... headers  
+
+- #1045 #1041 #1038 #1035 #1034 #1029 #1025 various cleanups and doc
+  improvments 
+
+## 1.0.0-beta.16
+
+- alleviate login problems with providers which only
+  support RSA1024 keys by switching back from Rustls 
+  to native-tls, by using the new async-email/async-native-tls 
+  crate from @dignifiedquire. thanks @link2xt. 
+
+- introduce per-contact profile images to send out 
+  own profile image heuristically, and fix sending
+  out of profile images in "in-prepare" groups. 
+  this also extends the Chat-spec that is maintained
+  in core to specify Chat-Group-Image and Chat-Group-Avatar
+  headers. thanks @r10s and @hpk42.
+
+- fix merging of protected headers from the encrypted
+  to the unencrypted parts, now not happening recursively
+  anymore.  thanks @hpk and @r10s
+
+- fix/optimize autocrypt gossip headers to only get 
+  sent when there are more than 2 people in a chat. 
+  thanks @link2xt
+
+- fix displayname to use the authenticated name 
+  when available (displayname as coming from contacts 
+  themselves). thanks @simon-laux
+
+- introduce preliminary support for offline autoconfig 
+  for nauta provider. thanks @hpk42 @r10s
+
+## 1.0.0-beta.15
+
+- fix #994 attachment appeared doubled in chats (and where actually
+  downloaded after smtp-send). @hpk42
+
+## 1.0.0-beta.14
+
+- fix packaging issue with our rust-email fork, now we are tracking
+  master again there. hpk42
+
+## 1.0.0-beta.13
+
+- fix #976 -- unicode-issues in display-name of email addresses. @hpk42
+
+- fix #985 group add/remove member bugs resulting in broken groups.  @hpk42
+
+- fix hanging IMAP connections -- we now detect with a 15second timeout
+  if we cannot terminate the IDLE IMAP protocol. @hpk42 @link2xt
+
+- fix incoming multipart/mixed containing html, to show up as
+  attachments again.  Fixes usage for simplebot which sends html
+  files for users to interact with the bot. @adbenitez @hpk42 
+
+- refinements to internal autocrypt-handling code, do not send
+  prefer-encrypt=nopreference as it is the default if no attribute
+  is present.  @linkxt 
+
+- simplify, modularize and rustify several parts 
+  of dc-core (general WIP). @link2xt @flub @hpk42 @r10s
+
+- use async-email/async-smtp to handle SMTP connections, might
+  fix connection/reconnection issues. @link2xt 
+
+- more tests and refinements for dealing with blobstorage @flub @hpk42 
+
+- use a dedicated build-server for CI testing of core PRs
+
+
+## 1.0.0-beta.12
+
+- fix python bindings to use core for copying attachments to blobdir
+  and fix core to actually do it. @hpk42
+
+## 1.0.0-beta.11
+
+- trigger reconnect more often on imap error states.  Should fix an 
+  issue observed when trying to empty a folder.  @hpk42
+
+- un-split qr tests: we fixed qr-securejoin protocol flakyness 
+  last weeks. @hpk42
+
+## 1.0.0-beta.10
+
+- fix grpid-determination from in-reply-to and references headers. @hpk42
+
+- only send Autocrypt-gossip headers on encrypted messages. @dignifiedquire
+
+- fix reply-to-encrypted message to also be encrypted. @hpk42
+
+- remove last unsafe code from dc_receive_imf :) @hpk42
+
+- add experimental new dc_chat_get_info_json FFI/API so that desktop devs
+  can play with using it. @jikstra
+
+- fix encoding of subjects and attachment-filenames @hpk42
+  @dignifiedquire . 
+
 ## 1.0.0-beta.9
 
 - historic: we now use the mailparse crate and lettre-email to generate mime
@@ -22,7 +334,9 @@
 - fix flakyness/sometimes-failing verified/join-protocols, 
   thanks @flub, @r10s, @hpk42
 
-- new DC_EVENT_SECUREJOIN_SUCCEEDED event 
+- fix reply-to-encrypted message to keep encryption 
+
+- new DC_EVENT_SECUREJOIN_MEMBER_ADDED event 
 
 - many little fixes and rustifications (@link2xt, @flub, @hpk42)
 

Cargo.toml

@@ -1,41 +1,42 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.8"
+version = "1.29.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
-license = "MPL"
+license = "MPL-2.0"
+
+[profile.release]
+lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { git = "https://github.com/rpgp/rpgp", branch = "master", default-features = false }
+pgp = { version = "0.5.1", default-features = false } 
 hex = "0.4.0"
 sha2 = "0.8.0"
-rand = "0.6.5"
-smallvec = "0.6.9"
-reqwest = { version = "0.9.15", default-features = false, features = ["rustls-tls"] }
-num-derive = "0.2.5"
+rand = "0.7.0"
+smallvec = "1.0.0"
+reqwest = { version = "0.10.0", features = ["blocking", "json"] }
+num-derive = "0.3.0"
 num-traits = "0.2.6"
-lettre = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
-async-imap = { git = "https://github.com/async-email/async-imap", branch="master" }
-async-tls = "0.6"
-async-std = { version = "1.0", features = ["unstable"] }
+async-smtp = "0.2"
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
+async-imap = "0.2"
+async-native-tls = "0.3.1"
+async-std = { version = "1.4", features = ["unstable"] }
 base64 = "0.11"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
-failure = "0.1.5"
-failure_derive = "0.1.5"
-# TODO: make optional
-rustyline = "4.1.0"
+indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.20", features = ["bundled"] }
-r2d2_sqlite = "0.12.0"
+rusqlite = { version = "0.21", features = ["bundled"] }
+r2d2_sqlite = "0.13.0"
 r2d2 = "0.8.5"
 strum = "0.16.0"
 strum_macros = "0.16.0"
@@ -44,16 +45,21 @@ backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.15.0"
+quick-xml = "0.17.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
 debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-rustls = "0.16.0"
-webpki-roots = "0.18.0"
-webpki = "0.21.0"
-mailparse = "0.10.1"
+mailparse = "0.12.0"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+native-tls = "0.2.3"
+image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+pretty_env_logger = "0.3.1"
+
+rustyline = { version = "4.1.0", optional = true }
+thiserror = "1.0.14"
+anyhow = "1.0.28"
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -74,10 +80,11 @@ path = "examples/simple.rs"
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
+required-features = ["rustyline"]
 
 
 [features]
-default = ["nightly", "ringbuf"]
-vendored = []
+default = ["nightly"]
+vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
-ringbuf = ["pgp/ringbuf"]
+

LICENSE

@@ -2,9 +2,6 @@ The files in this directory and under its subdirectories
 are (c) 2019 by Bjoern Petersen and contributors and released under the
 Mozilla Public License Version 2.0, see below for a copy.
 
-NOTE that the files in the "libs" directory are copyrighted by third parties
-and come with their own respective licenses.
-
 Mozilla Public License Version 2.0
 ==================================
 

README.md

@@ -17,8 +17,9 @@ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-cargo run --example repl -- /path/to/db
+cargo run --example repl -- ~/deltachat-db
 ```
+where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 
@@ -87,6 +88,15 @@ $ cargo test --all
 $ cargo build -p deltachat_ffi --release
 ```
 
+## Debugging environment variables 
+
+- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
+  printed
+
+- `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
+
+
+
 ### Expensive tests
 
 Some tests are expensive and marked with `#[ignore]`, to run these
@@ -99,9 +109,27 @@ $ cargo test -- --ignored
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
-- `ringbuf`: Enable the use of [`slice_deque`](https://github.com/gnzlbg/slice_deque) in pgp.
 
 [circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
 [circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
 [appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
 [appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
+## Language bindings and frontend projects
+
+Language bindings are available for:
+
+- [C](https://c.delta.chat)
+- [Node.js](https://www.npmjs.com/package/deltachat-node)
+- [Python](https://py.delta.chat)
+- [Go](https://github.com/hugot/go-deltachat/)
+- **Java** and **Swift** (contained in the Android/iOS repos)
+
+The following "frontend" projects make use of the Rust-library
+or its language bindings:
+
+- [Android](https://github.com/deltachat/deltachat-android)
+- [iOS](https://github.com/deltachat/deltachat-ios)
+- [Desktop](https://github.com/deltachat/deltachat-desktop)
+- [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- several **Bots**

appveyor.yml

@@ -4,11 +4,10 @@ environment:
 
 install:
   - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2019-07-10
+  - rustup-init -yv --default-toolchain nightly-2020-03-12
   - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
   - rustc -vV
   - cargo -vV
-  - cargo update
 
 build: false
 

assets/icon-device.svg

@@ -7,42 +7,42 @@
    xmlns="http://www.w3.org/2000/svg"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   inkscape:version="0.92.4 5da689c313, 2019-01-14"
-   sodipodi:docname="icon-device.svg"
-   id="svg4344"
-   preserveAspectRatio="xMidYMid meet"
-   viewBox="0 0 45 45"
-   height="60"
-   width="60"
-   version="1.0"
-   inkscape:export-filename="/home/kerle/Workspace/deltachat-core-rust/assets/icon-device.png"
+   inkscape:export-ydpi="409.60001"
    inkscape:export-xdpi="409.60001"
-   inkscape:export-ydpi="409.60001">
+   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-device.png"
+   version="1.0"
+   width="60"
+   height="60"
+   viewBox="0 0 45 45"
+   preserveAspectRatio="xMidYMid meet"
+   id="svg4344"
+   sodipodi:docname="icon-device.svg"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
   <defs
      id="defs4348" />
   <sodipodi:namedview
-     inkscape:current-layer="svg4344"
-     inkscape:window-maximized="0"
-     inkscape:window-y="0"
-     inkscape:window-x="0"
-     inkscape:cy="28.471578"
-     inkscape:cx="28.03926"
-     inkscape:zoom="5.6"
-     units="px"
-     showgrid="false"
-     id="namedview4346"
-     inkscape:window-height="1019"
-     inkscape:window-width="1248"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0"
-     guidetolerance="10"
-     gridtolerance="10"
-     objecttolerance="10"
-     borderopacity="1"
-     inkscape:document-rotation="0"
-     bordercolor="#666666"
+     inkscape:snap-global="false"
      pagecolor="#ffffff"
-     inkscape:snap-global="false" />
+     bordercolor="#666666"
+     inkscape:document-rotation="0"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1600"
+     inkscape:window-height="1035"
+     id="namedview4346"
+     showgrid="false"
+     units="px"
+     inkscape:zoom="3.959798"
+     inkscape:cx="28.322498"
+     inkscape:cy="24.898474"
+     inkscape:window-x="45"
+     inkscape:window-y="23"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4344" />
   <metadata
      id="metadata4336">
 Created by potrace 1.15, written by Peter Selinger 2001-2017
@@ -56,35 +56,28 @@ Created by potrace 1.15, written by Peter Selinger 2001-2017
   </cc:Work>
 </rdf:RDF>
 </metadata>
-  <circle
-     style="opacity:1;fill:#ffffff;fill-opacity:1;stroke-width:0.731071"
-     id="path859"
-     cx="22.5"
-     cy="22.5"
-     r="22.5" />
-  <circle
-     r="22.5"
-     cy="22.5"
-     cx="22.5"
-     id="path4917"
-     style="fill:#364e54;fill-opacity:0.67681497;stroke-width:0.58855402"
-     inkscape:export-xdpi="409"
-     inkscape:export-ydpi="409" />
+  <rect
+     y="-4.4408921e-16"
+     x="0"
+     height="45"
+     width="45"
+     id="rect860"
+     style="opacity:1;fill:#76868b;fill-opacity:1;stroke-width:0.819271" />
   <g
-     id="g4342"
-     transform="matrix(0.00255113,0,0,-0.00255113,5.586152,38.200477)"
-     style="fill:#ffffff;fill-opacity:1"
+     fill="#000000"
      stroke="none"
-     fill="#000000">
+     style="fill:#ffffff;fill-opacity:1"
+     transform="matrix(0.00255113,0,0,-0.00255113,5.586152,38.200477)"
+     id="g4342">
     <path
-       inkscape:connector-curvature="0"
-       id="path4338"
+       style="fill:#ffffff;fill-opacity:1"
        d="m 8175,12765 c -703,-114 -1248,-608 -1387,-1258 -17,-82 -21,-136 -22,-277 0,-202 15,-307 70,-470 149,-446 499,-733 1009,-828 142,-26 465,-23 619,6 691,131 1201,609 1328,1244 31,158 31,417 0,565 -114,533 -482,889 -1038,1004 -133,27 -448,35 -579,14 z"
-       style="fill:#ffffff;fill-opacity:1" />
+       id="path4338"
+       inkscape:connector-curvature="0" />
     <path
-       inkscape:connector-curvature="0"
-       id="path4340"
+       style="fill:#ffffff;fill-opacity:1"
        d="m 7070,9203 c -212,-20 -275,-27 -397,-48 -691,-117 -1400,-444 -2038,-940 -182,-142 -328,-270 -585,-517 -595,-571 -911,-974 -927,-1181 -6,-76 11,-120 69,-184 75,-80 159,-108 245,-79 109,37 263,181 632,595 539,606 774,826 1035,969 135,75 231,105 341,106 82,1 94,-2 138,-27 116,-68 161,-209 122,-376 -9,-36 -349,-868 -757,-1850 -407,-982 -785,-1892 -838,-2021 -287,-694 -513,-1389 -615,-1889 -70,-342 -90,-683 -52,-874 88,-440 381,-703 882,-792 124,-23 401,-30 562,-16 783,69 1674,461 2561,1125 796,596 1492,1354 1607,1751 43,146 -33,308 -168,360 -61,23 -100,15 -173,-36 -105,-74 -202,-170 -539,-529 -515,-551 -762,-783 -982,-927 -251,-164 -437,-186 -543,-65 -56,64 -74,131 -67,247 13,179 91,434 249,815 135,324 1588,4102 1646,4280 106,325 151,561 159,826 9,281 -22,463 -112,652 -58,122 -114,199 -211,292 -245,233 -582,343 -1044,338 -91,-1 -181,-3 -200,-5 z"
-       style="fill:#ffffff;fill-opacity:1" />
+       id="path4340"
+       inkscape:connector-curvature="0" />
   </g>
 </svg>

assets/icon-saved-messages.svg

@@ -7,42 +7,42 @@
    xmlns="http://www.w3.org/2000/svg"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   inkscape:version="0.92.4 5da689c313, 2019-01-14"
-   sodipodi:docname="icon-saved-messages.svg"
-   id="svg4344"
-   preserveAspectRatio="xMidYMid meet"
-   viewBox="0 0 45 45"
-   height="60"
-   width="60"
-   version="1.0"
-   inkscape:export-filename="/home/kerle/test-icon.png"
+   inkscape:export-ydpi="409.60001"
    inkscape:export-xdpi="409.60001"
-   inkscape:export-ydpi="409.60001">
+   inkscape:export-filename="/home/kerle/test-icon.png"
+   version="1.0"
+   width="60"
+   height="60"
+   viewBox="0 0 45 45"
+   preserveAspectRatio="xMidYMid meet"
+   id="svg4344"
+   sodipodi:docname="icon-saved-messages.svg"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
   <defs
      id="defs4348" />
   <sodipodi:namedview
-     inkscape:lockguides="false"
-     inkscape:current-layer="svg4344"
-     inkscape:window-maximized="0"
-     inkscape:window-y="130"
-     inkscape:window-x="89"
-     inkscape:cy="43.28624"
-     inkscape:cx="-21.066324"
-     inkscape:zoom="4"
-     units="px"
-     showgrid="false"
-     id="namedview4346"
-     inkscape:window-height="855"
-     inkscape:window-width="1395"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0"
-     guidetolerance="10"
-     gridtolerance="10"
-     objecttolerance="10"
-     borderopacity="1"
-     inkscape:document-rotation="0"
+     pagecolor="#ffffff"
      bordercolor="#666666"
-     pagecolor="#ffffff" />
+     inkscape:document-rotation="0"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1395"
+     inkscape:window-height="855"
+     id="namedview4346"
+     showgrid="false"
+     units="px"
+     inkscape:zoom="4"
+     inkscape:cx="29.308676"
+     inkscape:cy="49.03624"
+     inkscape:window-x="89"
+     inkscape:window-y="108"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4344"
+     inkscape:lockguides="false" />
   <metadata
      id="metadata4336">
 Created by potrace 1.15, written by Peter Selinger 2001-2017
@@ -56,21 +56,16 @@ Created by potrace 1.15, written by Peter Selinger 2001-2017
   </cc:Work>
 </rdf:RDF>
 </metadata>
-  <circle
-     style="opacity:1;fill:#ffffff;fill-opacity:1;stroke-width:0.731071"
-     id="path859"
-     cx="22.5"
-     cy="22.5"
-     r="22.5" />
-  <circle
-     r="22.5"
-     cy="22.50293"
-     cx="22.5"
-     id="path4917"
-     style="fill:#87aade;fill-opacity:1;stroke-width:0.58855402" />
+  <rect
+     y="0"
+     x="0"
+     height="45"
+     width="45"
+     id="rect1420"
+     style="fill:#87aade;fill-opacity:1;stroke:none;stroke-width:0.968078" />
   <path
-     inkscape:connector-curvature="0"
-     d="M 13.5,7.5 V 39 h 0.08654 L 22.533801,29.370239 31.482419,39 h 0.01758 V 7.5 Z m 9.004056,4.108698 1.879508,4.876388 5.039514,0.359779 -3.879358,3.363728 1.227764,5.095749 -4.276893,-2.796643 -4.280949,2.788618 1.237229,-5.093073 -3.873949,-3.371754 5.040866,-0.350417 z"
+     id="rect846"
      style="fill:#ffffff;stroke-width:0.58409804"
-     id="rect846" />
+     d="M 13.5,7.5 V 39 h 0.08654 L 22.533801,29.370239 31.482419,39 h 0.01758 V 7.5 Z m 9.004056,4.108698 1.879508,4.876388 5.039514,0.359779 -3.879358,3.363728 1.227764,5.095749 -4.276893,-2.796643 -4.280949,2.788618 1.237229,-5.093073 -3.873949,-3.371754 5.040866,-0.350417 z"
+     inkscape:connector-curvature="0" />
 </svg>

ci_scripts/ci_upload.sh

@@ -14,24 +14,22 @@ DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
 
-# DISABLED: python docs to py.delta.chat
-#ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-#rsync -avz \
-#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-#  "$PYDOCDIR/html/" \
-#  delta@py.delta.chat:build/${BRANCH}
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
 
 # C docs to c.delta.chat
 ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
 rsync -avz \
+  --delete \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   "$DOXYDOCDIR/html/" \
   delta@c.delta.chat:build-c/${BRANCH}
 
-exit 0
-
-# OUTDATED -- for re-use from python release-scripts 
-
 echo -----------------------
 echo upload wheels 
 echo -----------------------
@@ -39,6 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
+pip3 install -U pip
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN
@@ -50,6 +49,9 @@ devpi use dc/$N_BRANCH || {
     devpi use dc/$N_BRANCH
 }
 devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*.whl
+devpi upload deltachat*
 
 popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/cleanup_devpi_indices.py

@@ -0,0 +1,69 @@
+"""
+Remove old "dc" indices except for master which always stays.
+
+"""
+from requests import Session
+import datetime
+import sys
+import subprocess
+
+MAXDAYS=7
+
+session = Session()
+session.headers["Accept"] = "application/json"
+
+
+def get_indexes(baseurl, username):
+    response = session.get(baseurl + username)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["indexes"]
+
+
+def get_projectnames(baseurl, username, indexname):
+    response = session.get(baseurl + username + "/" + indexname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["projects"]
+
+
+def get_release_dates(baseurl, username, indexname, projectname):
+    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    dates = set()
+    for value in result.values():
+        if "+links" not in value:
+            continue
+        for link in value["+links"]:
+            for log in link["log"]:
+                dates.add(tuple(log["when"]))
+    return dates
+
+
+def run():
+    baseurl = "https://m.devpi.net/"
+    username = "dc"
+    for indexname in get_indexes(baseurl, username):
+        projectnames = get_projectnames(baseurl, username, indexname)
+        if indexname == "master" or not indexname:
+            continue
+        assert projectnames == ["deltachat"]
+        for projectname in projectnames:
+            dates = get_release_dates(baseurl, username, indexname, projectname)
+            if not dates:
+                print(
+                    "%s has no releases" % (baseurl + username + "/" + indexname),
+                    file=sys.stderr)
+                date = datetime.datetime.now()
+            else:
+                date = datetime.datetime(*max(dates))
+            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
+                assert username and indexname
+                url = baseurl + username + "/" + indexname
+                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+
+
+
+if __name__ == '__main__':
+    run()

ci_scripts/docker-coredeps/Dockerfile

@@ -5,14 +5,11 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install OpenSSL
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
 
-# Install OpenSSL
-ADD deps/build_openssl.sh /builder/build_openssl.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
-
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # Install python tools (auditwheels,tox, ...)

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,6 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library
-#
-set -e -x
-
-# perform clean build of core and install 
-export TOXWORKDIR=.docker-tox
-
-# build core library
-
-cargo build --release -p deltachat_ffi
-
-# configure access to a base python and 
-# to several python interpreters needed by tox below
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-#
-# run python tests
-#
-
-if [ -n "$TESTS" ]; then 
-
-    echo ----------------
-    echo run python tests
-    echo ----------------
-
-    pushd python 
-    # first run all tests ...
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
-    popd
-fi
-
-
-if [ -n "$DOCS" ]; then 
-    echo -----------------------
-    echo generating python docs
-    echo -----------------------
-    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
-fi

ci_scripts/old/run-python.sh

@@ -46,6 +46,7 @@ if [ -n "$TESTS" ]; then
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
     unset DCC_PY_LIVECONFIG
+    unset DCC_NEW_TMP_EMAIL
     tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
     tox --workdir "$TOXWORKDIR" -e auditwheels
     popd

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,51 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux1*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -2,7 +2,7 @@
 
 export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
-export SSHTARGET=ci@b1.delta.chat
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 # we construct the BUILDDIR such that we can easily share the
 # CARGO_TARGET_DIR between runs ("..")
@@ -30,6 +30,7 @@ ssh $SSHTARGET <<_HERE
     export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
     #rm -rf virtualenv venv

ci_scripts/remote_tests_rust.sh

@@ -2,7 +2,7 @@
 
 export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
-export SSHTARGET=ci@b1.delta.chat
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 # we construct the BUILDDIR such that we can easily share the
 # CARGO_TARGET_DIR between runs ("..")

ci_scripts/run-doxygen.sh

@@ -3,5 +3,4 @@
 set -ex
 
 cd deltachat-ffi 
-doxygen 
-
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run-python-test.sh

@@ -6,7 +6,7 @@
 set -e -x
 
 # for core-building and python install step
-export DCC_RS_TARGET=release
+export DCC_RS_TARGET=debug
 export DCC_RS_DEV=`pwd`
 
 cd python

ci_scripts/run_all.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Build the Delta Chat Core Rust library, Python wheels and docs 
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# compile core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+ln -s /opt/python/cp38-cp38/bin/python3.8
+popd
+
+pushd python
+# prepare a clean tox run
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+mkdir -p $TOXWORKDIR
+
+# disable live-account testing to speed up test runs and wheel building
+# XXX we may switch on some live-tests on for better ensurances 
+# Note that the independent remote_tests_python step does all kinds of
+# live-testing already. 
+unset DCC_PY_LIVECONFIG
+unset DCC_NEW_TMP_EMAIL
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+popd
+
+
+echo -----------------------
+echo generating python docs
+echo -----------------------
+(cd python && tox --workdir "$TOXWORKDIR" -e doc)

deltachat-ffi/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.8"
+version = "1.29.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
-license = "MIT OR Apache-2.0"
+license = "MPL-2.0"
 
 keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
@@ -16,14 +16,14 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { path = "../", default-features = false }
-deltachat-provider-database = "0.2.1"
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
-failure = "0.1.6"
+serde_json = "1.0"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 
 [features]
-default = ["vendored", "nightly", "ringbuf"]
+default = ["vendored", "nightly"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
-ringbuf = ["deltachat/ringbuf"]

deltachat-ffi/src/dc_array.rs

@@ -2,7 +2,6 @@ use crate::location::Location;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
-#[allow(non_camel_case_types)]
 pub enum dc_array_t {
     Locations(Vec<Location>),
     Uint(Vec<u32>),

deltachat-ffi/src/providers.rs

@@ -1,93 +0,0 @@
-extern crate deltachat_provider_database;
-
-use std::ptr;
-
-use crate::string::{to_string_lossy, StrExt};
-use deltachat_provider_database::StatusState;
-
-#[no_mangle]
-pub type dc_provider_t = deltachat_provider_database::Provider;
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_new_from_domain(
-    domain: *const libc::c_char,
-) -> *const dc_provider_t {
-    match deltachat_provider_database::get_provider_info(&to_string_lossy(domain)) {
-        Some(provider) => provider,
-        None => ptr::null(),
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_new_from_email(
-    email: *const libc::c_char,
-) -> *const dc_provider_t {
-    let email = to_string_lossy(email);
-    let domain = deltachat_provider_database::get_domain_from_email(&email);
-    match deltachat_provider_database::get_provider_info(domain) {
-        Some(provider) => provider,
-        None => ptr::null(),
-    }
-}
-
-macro_rules! null_guard {
-    ($context:tt) => {
-        if $context.is_null() {
-            return ptr::null_mut() as *mut libc::c_char;
-        }
-    };
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_get_overview_page(
-    provider: *const dc_provider_t,
-) -> *mut libc::c_char {
-    null_guard!(provider);
-    format!(
-        "{}/{}",
-        deltachat_provider_database::PROVIDER_OVERVIEW_URL,
-        (*provider).overview_page
-    )
-    .strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_get_name(provider: *const dc_provider_t) -> *mut libc::c_char {
-    null_guard!(provider);
-    (*provider).name.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_get_markdown(
-    provider: *const dc_provider_t,
-) -> *mut libc::c_char {
-    null_guard!(provider);
-    (*provider).markdown.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_get_status_date(
-    provider: *const dc_provider_t,
-) -> *mut libc::c_char {
-    null_guard!(provider);
-    (*provider).status.date.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_get_status(provider: *const dc_provider_t) -> u32 {
-    if provider.is_null() {
-        return 0;
-    }
-    match (*provider).status.state {
-        StatusState::OK => 1,
-        StatusState::PREPARATION => 2,
-        StatusState::BROKEN => 3,
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_unref(_provider: *const dc_provider_t) {
-    ()
-}
-
-// TODO expose general provider overview url?

deltachat-ffi/src/string.rs

@@ -1,5 +1,5 @@
-use failure::Fail;
 use std::ffi::{CStr, CString};
+use std::ptr;
 
 /// Duplicates a string
 ///
@@ -8,7 +8,7 @@ use std::ffi::{CStr, CString};
 /// # Examples
 ///
 /// ```rust,norun
-/// use deltachat::dc_tools::{dc_strdup, to_string_lossy};
+/// use crate::string::{dc_strdup, to_string_lossy};
 /// unsafe {
 ///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
 ///     let str_a_copy = dc_strdup(str_a);
@@ -16,7 +16,7 @@ use std::ffi::{CStr, CString};
 ///     assert_ne!(str_a, str_a_copy);
 /// }
 /// ```
-pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
+unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
     let ret: *mut libc::c_char;
     if !s.is_null() {
         ret = libc::strdup(s);
@@ -30,13 +30,13 @@ pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
 }
 
 /// Error type for the [OsStrExt] trait
-#[derive(Debug, Fail, PartialEq)]
-pub enum CStringError {
+#[derive(Debug, PartialEq, thiserror::Error)]
+pub(crate) enum CStringError {
     /// The string contains an interior null byte
-    #[fail(display = "String contains an interior null byte")]
+    #[error("String contains an interior null byte")]
     InteriorNullByte,
     /// The string is not valid Unicode
-    #[fail(display = "String is not valid unicode")]
+    #[error("String is not valid unicode")]
     NotUnicode,
 }
 
@@ -65,7 +65,7 @@ pub enum CStringError {
 ///     let mut c_ptr: *mut libc::c_char = dc_strdup(path_c.as_ptr());
 /// }
 /// ```
-pub trait OsStrExt {
+pub(crate) trait OsStrExt {
     /// Convert a  [std::ffi::OsStr] to an [std::ffi::CString]
     ///
     /// This is useful to convert e.g. a [std::path::Path] to
@@ -130,27 +130,28 @@ fn os_str_to_c_string_unicode(
 }
 
 /// Convenience methods/associated functions for working with [CString]
-///
-/// This is helps transitioning from unsafe code.
-pub trait CStringExt {
-    /// Create a new [CString], yolo style
+trait CStringExt {
+    /// Create a new [CString], best effort
     ///
-    /// This unwrap the result, panicking when there are embedded NULL
-    /// bytes.
-    fn yolo<T: Into<Vec<u8>>>(t: T) -> CString {
-        CString::new(t).expect("String contains null byte, can not be CString")
+    /// Like the [to_string_lossy] this doesn't give up in the face of
+    /// bad input (embedded null bytes in this case) instead it does
+    /// the best it can by stripping the embedded null bytes.
+    fn new_lossy<T: Into<Vec<u8>>>(t: T) -> CString {
+        let mut s = t.into();
+        s.retain(|&c| c != 0);
+        CString::new(s).unwrap_or_default()
     }
 }
 
 impl CStringExt for CString {}
 
-/// Convenience methods to make transitioning from raw C strings easier.
+/// Convenience methods to turn strings into C strings.
 ///
 /// To interact with (legacy) C APIs we often need to convert from
 /// Rust strings to raw C strings.  This can be clumsy to do correctly
 /// and the compiler sometimes allows it in an unsafe way.  These
 /// methods make it more succinct and help you get it right.
-pub trait StrExt {
+pub(crate) trait Strdup {
     /// Allocate a new raw C `*char` version of this string.
     ///
     /// This allocates a new raw C string which must be freed using
@@ -167,14 +168,52 @@ pub trait StrExt {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl<T: AsRef<str>> StrExt for T {
+impl<T: AsRef<str>> Strdup for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::yolo(self.as_ref());
+        let tmp = CString::new_lossy(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-pub fn to_string_lossy(s: *const libc::c_char) -> String {
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
+impl Strdup for std::path::Path {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
+        dc_strdup(tmp.as_ptr())
+    }
+}
+
+/// Convenience methods to turn optional strings into C strings.
+///
+/// This is the same as the [Strdup] trait but a different trait name
+/// to work around the type system not allowing to implement [Strdup]
+/// for `Option<impl Strdup>` When we already have an [Strdup] impl
+/// for `AsRef<&str>`.
+///
+/// When the [Option] is [Option::Some] this behaves just like
+/// [Strdup::strdup], when it is [Option::None] a null pointer is
+/// returned.
+pub(crate) trait OptStrdup {
+    /// Allocate a new raw C `*char` version of this string, or NULL.
+    ///
+    /// See [Strdup::strdup] for details.
+    unsafe fn strdup(&self) -> *mut libc::c_char;
+}
+
+impl<T: AsRef<str>> OptStrdup for Option<T> {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        match self {
+            Some(s) => {
+                let tmp = CString::new_lossy(s.as_ref());
+                dc_strdup(tmp.as_ptr())
+            }
+            None => ptr::null_mut(),
+        }
+    }
+}
+
+pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
     if s.is_null() {
         return "".into();
     }
@@ -184,7 +223,7 @@ pub fn to_string_lossy(s: *const libc::c_char) -> String {
     cstr.to_string_lossy().to_string()
 }
 
-pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
+pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
     if s.is_null() {
         return None;
     }
@@ -205,7 +244,7 @@ pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 ///
 /// [Path]: std::path::Path
 #[cfg(not(target_os = "windows"))]
-pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     assert!(!s.is_null(), "cannot be used on null pointers");
     use std::os::unix::ffi::OsStrExt;
     unsafe {
@@ -217,7 +256,7 @@ pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 
 // as_path() implementation for windows, documented above.
 #[cfg(target_os = "windows")]
-pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     as_path_unicode(s)
 }
 
@@ -324,8 +363,14 @@ mod tests {
     }
 
     #[test]
-    fn test_cstring_yolo() {
-        assert_eq!(CString::new("hello").unwrap(), CString::yolo("hello"));
+    fn test_cstring_new_lossy() {
+        assert!(CString::new("hel\x00lo").is_err());
+        assert!(CString::new(String::from("hel\x00o")).is_err());
+        let r = CString::new("hello").unwrap();
+        assert_eq!(CString::new_lossy("hello"), r);
+        assert_eq!(CString::new_lossy("hel\x00lo"), r);
+        assert_eq!(CString::new_lossy(String::from("hello")), r);
+        assert_eq!(CString::new_lossy(String::from("hel\x00lo")), r);
     }
 
     #[test]
@@ -347,4 +392,19 @@ mod tests {
             assert_eq!(cmp, 0);
         }
     }
+
+    #[test]
+    fn test_strdup_opt_string() {
+        unsafe {
+            let s = Some("hello");
+            let c = s.strdup();
+            let cmp = strcmp(c, b"hello\x00" as *const u8 as *const libc::c_char);
+            free(c as *mut libc::c_void);
+            assert_eq!(cmp, 0);
+
+            let s: Option<&str> = None;
+            let c = s.strdup();
+            assert_eq!(c, ptr::null_mut());
+        }
+    }
 }

deltachat_derive/Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "deltachat_derive"
-version = "0.1.0"
-authors = ["Dmitry Bogatov <KAction@debian.org>"]
+version = "2.0.0"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
+license = "MPL-2.0"
 
 [lib]
 proc-macro = true
 
 [dependencies]
-syn = "0.14.4"
-quote = "0.6.3"
+syn = "1.0.13"
+quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -3,7 +3,6 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
-use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to

draft/group-sync.rst

@@ -0,0 +1,126 @@
+
+Problem: missing eventual group consistency
+--------------------------------------------
+
+If group members are concurrently adding new members,
+the new members will miss each other's additions, example:
+
+- Alice and Bob are in a two-member group
+
+- Alice adds Carol, concurrently Bob adds Doris
+
+- Carol will see a three-member group (Alice, Bob, Carol),
+  Doris will see a different three-member group (Alice, Bob, Doris),
+  and only Alice and Bob will have all four members.
+
+Note that for verified groups any mitigation mechanism likely
+needs to make all clients to know who originally added a member.
+
+
+solution: memorize+attach (possible encrypted) chat-meta mime messages
+----------------------------------------------------------------------
+
+For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
+
+
+- All Chat-Group-Member-Added/Removed messages are recorded in their
+  full raw (signed and encrypted) mime-format in the DB
+
+- If an incoming member-add/member-delete messages has a member list
+  which is, apart from the added/removed member, not consistent
+  with our own view, broadcast a "Chat-Group-Member-Correction" message to
+  all members, attaching the original added/removed mime-message for all mismatching
+  contacts.  If we have no relevant add/del information, don't send a
+  correction message out.
+
+- Upong receiving added/removed attachments we don't do the
+  check_consistency+correction message dance.
+  This avoids recursion problems and hard-to-reason-about chatter.
+
+Notes:
+
+- mechanism works for both encrypted and unencrypted add/del messages
+
+- we already have a "mime_headers" column in the DB for each incoming message.
+  We could extend it to also include the payload and store mime unconditionally
+  for member-added/removed messages.
+
+- multiple member-added/removed messages can be attached in a single
+  correction message
+
+- it is minimal on the number of overall messages to reach group consistency
+  (best-case: no extra messages, the ABCD case above: max two extra messages)
+
+- somewhat backward compatible: older clients will probably ignore
+  messages which are signed by someone who is not the outer From-address.
+
+- the correction-protocol also helps with dropped messages.  If a member
+  did not see a member-added/removed message, the next member add/removed
+  message in the group will likely heal group consistency for this member.
+
+- we can quite easily extend the mechanism to also provide the group-avatar or
+  other meta-information.
+
+Discussions of variants
+++++++++++++++++++++++++
+
+- instead of acting on MemberAdded/Removed message we could send
+  corrections for any received message that addresses inconsistent group members but
+  a) this would delay group-membership healing
+  b) could lead to a lot of members sending corrections
+
+- instead of broadcasting correction messages we could only send it to
+  the sender of the inconsistent member-added/removed message.
+  A receiver of such a correction message would then need to forward
+  the message to the members it thinks also have an inconsistent view.
+  This sounds complicated and error-prone.  Concretely, if Alice
+  receives Bob's "Member-added: Doris" message, then Alice
+  broadcasting the correction message with "Member-added: Carol"
+  would reach all four members, healing group consistency in one step.
+  If Bob meanwhile receives Alice's "Member-Added: Carol" message,
+  Bob would broadcast a correction message to all four members as well.
+  (Imagine a situation where Alice/Bob added Carol/Doris
+  while both being in an offline or bad-connection situation).
+
+
+solution2: repeat member-added/removed messages
+---------------------------------------------------
+
+Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
+but keep sending out the old headers until the new protocol is sufficiently deployed.
+
+The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
+which controls repetition of the signed "add/del e-mail address" payload.
+
+Example::
+
+    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
+        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
+        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
+        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
+        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
+        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
+        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
+        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
+        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
+        =P6GG
+
+TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
+Receivers will apply the add/del change to the group-membership,
+decrease the TTL by 1, and if TTL>0 re-sent the header.
+
+The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
+This allows to propagate, in a cryptographically secured way,
+who added a member. This is particularly important for allowing
+to show in verified groups who added a member (planned).
+
+Disadvantage to solution 1:
+
+- requires to specify encoding and precise rules for what/how is signed.
+
+- causes O(N^2) extra messages
+
+- Not easily extendable for other things (without introducing a new
+  header / encoding)
+
+

examples/repl/cmdline.rs

@@ -1,9 +1,9 @@
 use std::path::Path;
 use std::str::FromStr;
 
-use deltachat::chat::{self, Chat};
+use anyhow::{bail, ensure};
+use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
 use deltachat::chatlist::*;
-use deltachat::config;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
@@ -19,15 +19,16 @@ use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::Event;
+use deltachat::{config, provider};
 
-/// Reset database tables. This function is called from Core cmdline.
+/// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
 /// e.g. bitmask 7 triggers actions definded with bits 1, 2 and 4.
-pub fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
-    info!(context, "Resetting tables ({})...", bits);
+fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
+    println!("Resetting tables ({})...", bits);
     if 0 != bits & 1 {
         sql::execute(context, &context.sql, "DELETE FROM jobs;", params![]).unwrap();
-        info!(context, "(1) Jobs reset.");
+        println!("(1) Jobs reset.");
     }
     if 0 != bits & 2 {
         sql::execute(
@@ -37,11 +38,11 @@ pub fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
             params![],
         )
         .unwrap();
-        info!(context, "(2) Peerstates reset.");
+        println!("(2) Peerstates reset.");
     }
     if 0 != bits & 4 {
         sql::execute(context, &context.sql, "DELETE FROM keypairs;", params![]).unwrap();
-        info!(context, "(4) Private keypairs reset.");
+        println!("(4) Private keypairs reset.");
     }
     if 0 != bits & 8 {
         sql::execute(
@@ -80,21 +81,23 @@ pub fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
         )
         .unwrap();
         sql::execute(context, &context.sql, "DELETE FROM leftgrps;", params![]).unwrap();
-        info!(context, "(8) Rest but server config reset.");
+        println!("(8) Rest but server config reset.");
     }
 
     context.call_cb(Event::MsgsChanged {
-        chat_id: 0,
+        chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
 
     1
 }
 
-fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), Error> {
+fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
     let data = dc_read_file(context, filename)?;
 
-    dc_receive_imf(context, &data, "import", 0, 0);
+    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false) {
+        println!("dc_receive_imf errored: {:?}", err);
+    }
     Ok(())
 }
 
@@ -153,7 +156,7 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
                 let name = name_f.to_string_lossy();
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
-                    info!(context, "Import: {}", path_plus_name);
+                    println!("Import: {}", path_plus_name);
                     if dc_poke_eml_file(context, path_plus_name).is_ok() {
                         read_cnt += 1
                     }
@@ -161,13 +164,10 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
             }
         }
     }
-    info!(
-        context,
-        "Import: {} items read from \"{}\".", read_cnt, &real_spec
-    );
+    println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
         context.call_cb(Event::MsgsChanged {
-            chat_id: 0,
+            chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
     }
@@ -188,8 +188,7 @@ fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     };
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
-    info!(
-        context,
+    println!(
         "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
@@ -223,16 +222,14 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id.is_daymarker() {
-            info!(
-                context,
+            println!(
                 "--------------------------------------------------------------------------------"
             );
 
             lines_out += 1
         } else if !msg_id.is_special() {
             if lines_out == 0 {
-                info!(
-                    context,
+                println!(
                     "--------------------------------------------------------------------------------",
                 );
                 lines_out += 1
@@ -242,8 +239,7 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
         }
     }
     if lines_out > 0 {
-        info!(
-            context,
+        println!(
             "--------------------------------------------------------------------------------"
         );
     }
@@ -293,7 +289,7 @@ fn log_contactlist(context: &Context, contacts: &Vec<u32>) {
                 );
             }
 
-            info!(context, "Contact#{}: {}{}", contact_id, line, line2);
+            println!("Contact#{}: {}{}", contact_id, line, line2);
         }
     }
 }
@@ -302,9 +298,9 @@ fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
+pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
     let chat_id = *context.cmdline_sel_chat_id.read().unwrap();
-    let mut sel_chat = if chat_id > 0 {
+    let mut sel_chat = if !chat_id.is_unset() {
         Chat::load_from_db(context, chat_id).ok()
     } else {
         None
@@ -376,6 +372,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                  listmedia\n\
                  archive <chat-id>\n\
                  unarchive <chat-id>\n\
+                 pin <chat-id>\n\
+                 unpin <chat-id>\n\
                  delchat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
@@ -397,8 +395,10 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
+                 providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
+                 estimatedeletion <seconds>\n\
                  emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
@@ -489,7 +489,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             println!("{:#?}", context.get_info());
         }
         "interrupt" => {
-            interrupt_inbox_idle(context, true);
+            interrupt_inbox_idle(context);
         }
         "maybenetwork" => {
             maybe_network(context);
@@ -508,23 +508,26 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
 
             let cnt = chatlist.len();
             if cnt > 0 {
-                info!(
-                    context,
+                println!(
                     "================================================================================"
                 );
 
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
-                    info!(
-                        context,
-                        "{}#{}: {} [{} fresh]",
+                    println!(
+                        "{}#{}: {} [{} fresh] {}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat::get_fresh_msg_cnt(context, chat.get_id()),
+                        chat.get_id().get_fresh_msg_cnt(context),
+                        match chat.visibility {
+                            ChatVisibility::Normal => "",
+                            ChatVisibility::Archived => "📦",
+                            ChatVisibility::Pinned => "📌",
+                        },
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
-                    let statestr = if chat.is_archived() {
+                    let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
                         match lot.get_state() {
@@ -538,8 +541,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                     let timestr = dc_timestamp_to_str(lot.get_timestamp());
                     let text1 = lot.get_text1();
                     let text2 = lot.get_text2();
-                    info!(
-                        context,
+                    println!(
                         "{}{}{}{} [{}]{}",
                         text1.unwrap_or(""),
                         if text1.is_some() { ": " } else { "" },
@@ -552,14 +554,13 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                             ""
                         },
                     );
-                    info!(
-                        context,
+                    println!(
                         "================================================================================"
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(context, 0) {
-                info!(context, "Location streaming enabled.");
+            if location::is_sending_locations_to_chat(context, ChatId::new(0)) {
+                println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
         }
@@ -568,8 +569,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 bail!("Argument [chat-id] is missing.");
             }
             if !arg1.is_empty() {
-                let chat_id = arg1.parse()?;
-                println!("Selecting chat #{}", chat_id);
+                let chat_id = ChatId::new(arg1.parse()?);
+                println!("Selecting chat {}", chat_id);
                 sel_chat = Some(Chat::load_from_db(context, chat_id)?);
                 *context.cmdline_sel_chat_id.write().unwrap() = chat_id;
             }
@@ -587,8 +588,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             } else {
                 format!("{} member(s)", members.len())
             };
-            info!(
-                context,
+            println!(
                 "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
@@ -608,14 +608,11 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 },
             );
             log_msglist(context, &msglist)?;
-            if let Some(draft) = chat::get_draft(context, sel_chat.get_id())? {
+            if let Some(draft) = sel_chat.get_id().get_draft(context)? {
                 log_msg(context, "Draft", &draft);
             }
 
-            println!(
-                "{} messages.",
-                chat::get_msg_cnt(context, sel_chat.get_id())
-            );
+            println!("{} messages.", sel_chat.get_id().get_msg_cnt(context));
             chat::marknoticed_chat(context, sel_chat.get_id())?;
         }
         "createchat" => {
@@ -691,7 +688,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let contacts = chat::get_chat_contacts(context, sel_chat.as_ref().unwrap().get_id());
-            info!(context, "Memberlist:");
+            println!("Memberlist:");
 
             log_contactlist(context, &contacts);
             println!(
@@ -717,8 +714,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
-                info!(
-                    context,
+                println!(
                     "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
                     dc_timestamp_to_str(location.timestamp),
@@ -732,7 +728,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 );
             }
             if locations.is_empty() {
-                info!(context, "No locations.");
+                println!("No locations.");
             }
         }
         "sendlocations" => {
@@ -798,7 +794,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             let chat = if let Some(ref sel_chat) = sel_chat {
                 sel_chat.get_id()
             } else {
-                0 as libc::c_uint
+                ChatId::new(0)
             };
 
             let msglist = context.search_msgs(chat, arg1);
@@ -812,14 +808,14 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             if !arg1.is_empty() {
                 let mut draft = Message::new(Viewtype::Text);
                 draft.set_text(Some(arg1.to_string()));
-                chat::set_draft(
-                    context,
-                    sel_chat.as_ref().unwrap().get_id(),
-                    Some(&mut draft),
-                );
+                sel_chat
+                    .as_ref()
+                    .unwrap()
+                    .get_id()
+                    .set_draft(context, Some(&mut draft));
                 println!("Draft saved.");
             } else {
-                chat::set_draft(context, sel_chat.as_ref().unwrap().get_id(), None);
+                sel_chat.as_ref().unwrap().get_id().set_draft(context, None);
                 println!("Draft deleted.");
             }
         }
@@ -855,15 +851,23 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             }
             print!("\n");
         }
-        "archive" | "unarchive" => {
+        "archive" | "unarchive" | "pin" | "unpin" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = arg1.parse()?;
-            chat::archive(context, chat_id, arg0 == "archive")?;
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id.set_visibility(
+                context,
+                match arg0 {
+                    "archive" => ChatVisibility::Archived,
+                    "unarchive" | "unpin" => ChatVisibility::Normal,
+                    "pin" => ChatVisibility::Pinned,
+                    _ => panic!("Unexpected command (This should never happen)"),
+                },
+            )?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = arg1.parse()?;
-            chat::delete(context, chat_id)?;
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id.delete(context)?;
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -879,12 +883,12 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
         }
         "forward" => {
             ensure!(
-                !arg1.is_empty() && arg2.is_empty(),
+                !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <chat-id> expected"
             );
 
             let mut msg_ids = [MsgId::new(0); 1];
-            let chat_id = arg2.parse()?;
+            let chat_id = ChatId::new(arg2.parse()?);
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(context, &msg_ids, chat_id)?;
         }
@@ -936,7 +940,14 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             let contact = Contact::get_by_id(context, contact_id)?;
             let name_n_addr = contact.get_name_n_addr();
 
-            let mut res = format!("Contact info for: {}:\n\n", name_n_addr);
+            let mut res = format!(
+                "Contact info for: {}:\nIcon: {}\n",
+                name_n_addr,
+                match contact.get_profile_image(context) {
+                    Some(image) => image.to_str().unwrap().to_string(),
+                    None => "NoIcon".to_string(),
+                }
+            );
 
             res += &Contact::get_encrinfo(context, contact_id)?;
 
@@ -973,6 +984,31 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 res.get_text2()
             );
         }
+        "setqr" => {
+            ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
+            match set_config_from_qr(context, arg1) {
+                Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
+                Err(err) => println!("Cannot set config from QR code: {:?}", err),
+            }
+        }
+        "providerinfo" => {
+            ensure!(!arg1.is_empty(), "Argument <addr> missing.");
+            match provider::get_provider_info(arg1) {
+                Some(info) => {
+                    println!("Information for provider belonging to {}:", arg1);
+                    println!("status: {}", info.status as u32);
+                    println!("before_login_hint: {}", info.before_login_hint);
+                    println!("after_login_hint: {}", info.after_login_hint);
+                    println!("overview_page: {}", info.overview_page);
+                    for server in info.server.iter() {
+                        println!("server: {}:{}", server.hostname, server.port,);
+                    }
+                }
+                None => {
+                    println!("No information for provider belonging to {} found.", arg1);
+                }
+            }
+        }
         // TODO: implement this again, unclear how to match this through though, without writing a parser.
         // "event" => {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
@@ -994,6 +1030,16 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 bail!("Command failed.");
             }
         }
+        "estimatedeletion" => {
+            ensure!(!arg1.is_empty(), "Argument <seconds> missing");
+            let seconds = arg1.parse()?;
+            let device_cnt = message::estimate_deletion_cnt(context, false, seconds)?;
+            let server_cnt = message::estimate_deletion_cnt(context, true, seconds)?;
+            println!(
+                "estimated count of messages older than {} seconds:\non device: {}\non server: {}",
+                seconds, device_cnt, server_cnt
+            );
+        }
         "emptyserver" => {
             ensure!(!arg1.is_empty(), "Argument <flags> missing");
 

examples/repl/main.rs

@@ -7,8 +7,6 @@
 #[macro_use]
 extern crate deltachat;
 #[macro_use]
-extern crate failure;
-#[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate rusqlite;
@@ -20,8 +18,9 @@ use std::process::Command;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 
+use anyhow::{bail, Error};
+use deltachat::chat::ChatId;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::context::*;
 use deltachat::job::*;
 use deltachat::oauth2::*;
@@ -41,7 +40,7 @@ use self::cmdline::*;
 
 // Event Handler
 
-fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
+fn receive_event(_context: &Context, event: Event) {
     match event {
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
@@ -111,8 +110,6 @@ fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
             print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
         }
     }
-
-    0
 }
 
 // Threads for waiting for messages and for jobs
@@ -202,7 +199,7 @@ fn stop_threads(context: &Context) {
         println!("Stopping threads");
         IS_RUNNING.store(false, Ordering::Relaxed);
 
-        interrupt_inbox_idle(context, true);
+        interrupt_inbox_idle(context);
         interrupt_mvbox_idle(context);
         interrupt_sentbox_idle(context);
         interrupt_smtp_idle(context);
@@ -264,7 +261,7 @@ const DB_COMMANDS: [&str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 24] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
@@ -288,6 +285,8 @@ const CHAT_COMMANDS: [&str; 24] = [
     "listmedia",
     "archive",
     "unarchive",
+    "pin",
+    "unpin",
     "delchat",
 ];
 const MESSAGE_COMMANDS: [&str; 8] = [
@@ -308,8 +307,17 @@ const CONTACT_COMMANDS: [&str; 6] = [
     "delcontact",
     "cleanupcontacts",
 ];
-const MISC_COMMANDS: [&str; 9] = [
-    "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
+const MISC_COMMANDS: [&str; 10] = [
+    "getqr",
+    "getbadqr",
+    "checkqr",
+    "event",
+    "fileinfo",
+    "clear",
+    "exit",
+    "quit",
+    "help",
+    "estimatedeletion",
 ];
 
 impl Hinter for DcHelper {
@@ -361,10 +369,10 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
+fn main_0(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
-        return Err(format_err!("No db-name specified"));
+        bail!("No db-name specified");
     }
     let context = Context::new(
         Box::new(receive_event),
@@ -434,7 +442,7 @@ enum ExitResult {
     Exit,
 }
 
-fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failure::Error> {
+fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
@@ -462,7 +470,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
         }
         "configure" => {
             start_threads(ctx.clone());
-            configure(&ctx.read().unwrap());
+            ctx.read().unwrap().configure();
         }
         "oauth2" => {
             if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
@@ -486,9 +494,10 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
         }
         "getqr" | "getbadqr" => {
             start_threads(ctx.clone());
-            if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx.read().unwrap(), arg1.parse().unwrap_or_default())
-            {
+            if let Some(mut qr) = dc_get_securejoin_qr(
+                &ctx.read().unwrap(),
+                ChatId::new(arg1.parse().unwrap_or_default()),
+            ) {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -516,7 +525,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
     Ok(ExitResult::Continue)
 }
 
-pub fn main() -> Result<(), failure::Error> {
+pub fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
     let args: Vec<String> = std::env::args().collect();

examples/simple.rs

@@ -7,7 +7,6 @@ use tempfile::tempdir;
 use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::job::{
@@ -16,21 +15,18 @@ use deltachat::job::{
 };
 use deltachat::Event;
 
-fn cb(_ctx: &Context, event: Event) -> usize {
+fn cb(_ctx: &Context, event: Event) {
     print!("[{:?}]", event);
 
     match event {
         Event::ConfigureProgress(progress) => {
-            print!("  progress: {}\n", progress);
-            0
+            println!("  progress: {}", progress);
         }
         Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            print!("  {}\n", msg);
-            0
+            println!("  {}", msg);
         }
         _ => {
-            print!("\n");
-            0
+            println!();
         }
     }
 }
@@ -80,7 +76,7 @@ fn main() {
     ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
         .unwrap();
     ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-    configure(&ctx);
+    ctx.configure();
 
     thread::sleep(duration);
 
@@ -104,7 +100,7 @@ fn main() {
     println!("stopping threads");
 
     *running.write().unwrap() = false;
-    deltachat::job::interrupt_inbox_idle(&ctx, true);
+    deltachat::job::interrupt_inbox_idle(&ctx);
     deltachat::job::interrupt_smtp_idle(&ctx);
 
     println!("joining");

python/CHANGELOG

@@ -1,6 +1,29 @@
-0.600.1
+0.900.0 (DRAFT)
+---------------
+
+- refactored internals to use plugin-approach 
+
+- introduced PerAccount and Global hooks that plugins can implement 
+
+- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
+
+- introduced two documented examples for an echo and a group-membership
+  tracking plugin. 
+
+0.800.0
+-------
+
+- use latest core 1.25.0 
+
+- refine tests and some internal changes to core bindings
+
+0.700.0
 ---------
 
+- lots of new Python APIs
+
+- use rust core-beta23 
+
 - introduce automatic versioning via setuptools_scm, 
   based on py-X.Y.Z tags 
 

python/README.rst

@@ -3,41 +3,98 @@ deltachat python bindings
 =========================
 
 This package provides bindings to the deltachat-core_ Rust -library
-which provides imap/smtp/crypto handling as well as chat/group/messages
-handling to Android, Desktop and IO user interfaces.
+which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
+a low-level Chat/Contact/Message API to user interfaces and bots.
 
-Installing pre-built packages (linux-only)
-==========================================
 
-If you have a linux system you may install the ``deltachat`` binary "wheel" package
+Installing bindings from source (Updated: 20-Jan-2020)
+=========================================================
+
+Install Rust and Cargo first.  Deltachat needs a specific nightly
+version, the easiest is probably to first install Rust stable from
+rustup and then use this to install the correct nightly version.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then GIT clone the deltachat-core-rust repo and get the actual
+rust- and cargo-toolchain needed by deltachat::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+   rustup show
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   ./install_python_bindings.py
+
+The installation might take a while, depending on your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_NEW_TMP_EMAIL`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs. This is implemented on the server side via
+  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
+Installing pre-built packages (Linux-only)
+========================================================
+
+If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
 
-1. `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
-   then create a fresh python environment and activate it in your shell::
+We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
         source venv/bin/activate
 
-   Afterwards, invoking ``python`` or ``pip install`` will only
-   modify files in your ``venv`` directory and leave your system installation
-   alone.
-
-2. Install the wheel for linux::
-
-        pip install deltachat
-
-    Verify it worked by typing::
-
-        python -c "import deltachat"
-
-
-Installing a wheel from a PR/branch
----------------------------------------
+Afterwards, invoking ``python`` or ``pip install`` only
+modifies files in your ``venv`` directory and leaves
+your system installation alone.
 
 For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest github ``master`` branch::
+and push them to a python package index. To install the latest
+github ``master`` branch::
 
-    pip install -i https://m.devpi.net/dc/master deltachat
+    pip install --pre -i https://m.devpi.net/dc/master deltachat
+
+To verify it worked::
+
+    python -c "import deltachat"
 
 .. note::
 
@@ -46,65 +103,6 @@ and push them to a python package index. To install the latest github ``master``
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
-Installing bindings from source
-===============================
-
-If you can't use "binary" method above then you need to compile
-to core deltachat library::
-
-    git clone https://github.com/deltachat/deltachat-core-rust
-    cd deltachat-core-rust
-    cd python
-
-If you don't have one active, create and activate a python "virtualenv":
-
-    python virtualenv venv  # or python -m venv
-    source venv/bin/activate
-
-Afterwards ``which python`` tells you that it comes out of the "venv"
-directory that contains all python install artifacts. Let's first
-install test tools::
-
-    pip install pytest pytest-timeout pytest-rerunfailures requests
-
-then cargo-build and install the deltachat bindings::
-
-    python install_python_bindings.py
-
-The bindings will be installed in release mode but with debug symbols.
-The release mode is necessary because some tests generate RSA keys
-which is prohibitively slow in debug mode.
-
-After successful binding installation you can finally run the tests::
-
-    pytest -v tests
-
-.. note::
-
-    Some tests are sometimes failing/hanging because of
-    https://github.com/deltachat/deltachat-core-rust/issues/331
-    and
-    https://github.com/deltachat/deltachat-core-rust/issues/326
-
-
-running "live" tests (experimental)
------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_PY_LIVECONFIG`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
-
 Code examples
 =============
 
@@ -115,15 +113,11 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux1 wheels
-==========================
-
-.. note::
-
-   This section may not fully work.
+Building manylinux1 based wheels
+================================
 
 Building portable manylinux1 wheels which come with libdeltachat.so
-and all it's dependencies is easy using the provided docker tooling.
+can be done with docker-tooling.
 
 using docker pull / premade images
 ------------------------------------
@@ -136,9 +130,9 @@ organization::
 
 This docker image can be used to run tests and build Python wheels for all interpreters::
 
-    $ bash ci_scripts/ci_run.sh
-
-This command runs tests and build-wheel scripts in a docker container.
+    $ docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image

python/doc/_static/custom.css

@@ -15,3 +15,7 @@ div.globaltoc {
 img.logo {
     height: 120px;
 }
+
+div.footer {
+    display: none;
+}

python/doc/api.rst

@@ -2,10 +2,6 @@
 high level API reference
 ========================
 
-.. note::
-
-    This API is work in progress and may change in versions prior to 1.0.
-
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
 - :class:`deltachat.contact.Contact`

python/doc/capi.rst

@@ -1,7 +0,0 @@
-
-C deltachat interface
-=====================
-
-See :doc:`lapi` for accessing many of the below functions
-through the ``deltachat.capi.lib`` namespace.
-

python/doc/conf.py

@@ -55,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'deltachat'
-copyright = u'2018, holger krekel and contributors'
+copyright = u'2020, holger krekel and contributors'
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation

python/doc/examples.rst

@@ -1,37 +1,60 @@
 
-
 examples
 ========
 
-
-Playing around on the commandline
-----------------------------------
-
 Once you have :doc:`installed deltachat bindings <install>`
-you can start playing from the python interpreter commandline.
-For example you can type ``python`` and then::
+you need email/password credentials for an IMAP/SMTP account.
+Delta Chat developers and the CI system use a special URL to create
+temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
 
-    # instantiate and configure deltachat account
-    import deltachat
-    ac = deltachat.Account("/tmp/db")
+Receiving a Chat message from the command line
+----------------------------------------------
 
-    # start configuration activity and smtp/imap threads
-    ac.start_threads()
-    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
+Here is a simple bot that:
 
-    # create a contact and send a message
-    contact = ac.create_contact("someother@email.address")
-    chat = ac.create_chat_by_contact(contact)
-    chat.send_text("hi from the python interpreter command line")
+- receives a message and sends back ("echoes") a message
 
-Checkout our :doc:`api` for the various high-level things you can do
-to send/receive messages, create contacts and chats.
+- terminates the bot if the message `/quit` is sent
 
+.. include:: ../examples/echo_and_quit.py
+    :literal:
 
-Looking at a real example
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    $ cd examples
+    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
+
+While this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Track member additions and removals in a group
+----------------------------------------------
+
+Here is a simple bot that:
+
+- echoes messages sent to it
+
+- tracks if configuration completed
+
+- tracks member additions and removals for all chat groups
+
+.. include:: ../examples/group_tracking.py
+    :literal:
+
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
+
+When this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Writing bots for real
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a real-life example of Python bindings usage.
-
+contains a little framework for writing deltachat bots in Python.
 

python/doc/index.rst

@@ -1,15 +1,15 @@
 deltachat python bindings
 =========================
 
-The ``deltachat`` Python package provides two bindings for the core Rust-library
-of the https://delta.chat messaging ecosystem:
+The ``deltachat`` Python package provides two layers of bindings for the
+core Rust-library of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core which aims
-  to be memory safe and thoroughly tested through continous tox/pytest runs.
+- :doc:`api` is a high level interface to deltachat-core.
 
-- :doc:`capi` is a lowlevel CFFI-binding to the previous
-  `deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
-  replicates exactly the same C-level API).
+- :doc:`plugins` is a brief introduction into implementing plugin hooks.
+
+- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
+  <https://github.com/deltachat/deltachat-core-rust>`_.
 
 
 
@@ -28,8 +28,8 @@ getting started
    links
    changelog
    api
-   capi
    lapi
+   plugins
 
 ..
     Indices and tables

python/doc/plugins.rst

@@ -0,0 +1,38 @@
+
+Implementing Plugin Hooks
+==========================
+
+The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
+for managing global and per-account plugin registration, and performing
+hook calls. There are two kinds of plugins:
+
+- Global plugins that are active for all accounts; they can implement
+  hooks at account-creation and account-shutdown time.
+
+- Account plugins that are only active during the lifetime of a
+  single Account instance.
+
+
+Registering a plugin
+--------------------
+
+.. autofunction:: deltachat.register_global_plugin
+    :noindex:
+
+.. automethod:: deltachat.account.Account.add_account_plugin
+    :noindex:
+
+
+Per-Account Hook specifications
+-------------------------------
+
+.. autoclass:: deltachat.hookspec.PerAccount
+    :members:
+
+
+Global Hook specifications
+--------------------------
+
+.. autoclass:: deltachat.hookspec.Global
+    :members:
+

python/examples/echo_and_quit.py

@@ -0,0 +1,30 @@
+
+# content of echo_and_quit.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class EchoPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_message_delivered(self, message):
+        print("ac_message_delivered", message)
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/group_tracking.py

@@ -0,0 +1,52 @@
+
+# content of group_tracking.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class GroupTrackingPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_outgoing_message(self, message):
+        print("ac_outgoing_message:", message)
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        print("ac_configure_completed:", success)
+
+    @account_hookimpl
+    def ac_chat_modified(self, chat):
+        print("ac_chat_modified:", chat.id, chat.get_name())
+        for member in chat.get_contacts():
+            print("chat member: {}".format(member.addr))
+
+    @account_hookimpl
+    def ac_member_added(self, chat, contact, message):
+        print("ac_member_added {} to chat {} from {}".format(
+            contact.addr, chat.id, message.get_sender_contact().addr))
+        for member in chat.get_contacts():
+            print("chat member: {}".format(member.addr))
+
+    @account_hookimpl
+    def ac_member_removed(self, chat, contact, message):
+        print("ac_member_removed {} from chat {} by {}".format(
+            contact.addr, chat.id, message.get_sender_contact().addr))
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/test_examples.py

@@ -0,0 +1,72 @@
+
+import pytest
+import py
+import echo_and_quit
+import group_tracking
+from deltachat.eventlogger import FFIEventLogger
+
+
+@pytest.fixture(scope='session')
+def datadir():
+    """The py.path.local object of the test-data/ directory."""
+    for path in reversed(py.path.local(__file__).parts()):
+        datadir = path.join('test-data')
+        if datadir.isdir():
+            return datadir
+    else:
+        pytest.skip('test-data directory not found')
+
+
+def test_echo_quit_plugin(acfactory):
+    botproc = acfactory.run_bot_process(echo_and_quit)
+
+    ac1 = acfactory.get_one_online_account()
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch1 = ac1.create_chat_by_contact(bot_contact)
+    ch1.send_text("hello")
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+    assert reply.chat == ch1
+    ch1.send_text("/quit")
+    botproc.wait()
+
+
+def test_group_tracking_plugin(acfactory, lp):
+    lp.sec("creating one group-tracking bot and two temp accounts")
+    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
+
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
+
+    botproc.fnmatch_lines("""
+        *ac_configure_completed*
+    """)
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+
+    lp.sec("creating bot test group with bot")
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch = ac1.create_group_chat("bot test group")
+    ch.add_contact(bot_contact)
+    ch.send_text("hello")
+
+    botproc.fnmatch_lines("""
+        *ac_chat_modified*bot test group*
+    """.format(ac1.get_config("addr")))
+
+    lp.sec("adding third member {}".format(ac2.get_config("addr")))
+    contact3 = ac1.create_contact(ac2.get_config("addr"))
+    ch.add_contact(contact3)
+
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+
+    lp.sec("now looking at what the bot received")
+    botproc.fnmatch_lines("""
+        *ac_member_added {}*
+    """.format(contact3.addr))
+
+    lp.sec("contact successfully added, now removing")
+    ch.remove_contact(contact3)
+    botproc.fnmatch_lines("""
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/install_python_bindings.py

@@ -11,18 +11,15 @@ import sys
 if __name__ == "__main__":
     target = os.environ.get("DCC_RS_TARGET")
     if target is None:
-        os.environ["DCC_RS_TARGET"] = target = "release"
+        os.environ["DCC_RS_TARGET"] = target = "debug"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
-    # build the core library  in release + debug mode because
-    # as of Nov 2019 rPGP generates RSA keys which take
-    # prohibitively long for non-release installs
-    os.environ["RUSTFLAGS"] = "-g"
-    subprocess.check_call([
-        "cargo", "build", "-p", "deltachat_ffi", "--" + target
-    ])
+    cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+    if target == 'release':
+        cmd.append("--release")
+    subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":

python/setup.py

@@ -13,14 +13,20 @@ def main():
             "root": "..",
             "relative_to": __file__,
             'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
         },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'six'],
+        install_requires=['cffi>=1.0.0', 'pluggy'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
+        entry_points = {
+            'pytest11': [
+                'deltachat.testplugin = deltachat.testplugin',
+            ],
+        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,6 +1,13 @@
-from deltachat import capi, const
-from deltachat.capi import ffi
-from deltachat.account import Account  # noqa
+import sys
+
+from . import capi, const, hookspec
+from .capi import ffi
+from .account import Account  # noqa
+from .message import Message  # noqa
+from .contact import Contact  # noqa
+from .chat import Chat        # noqa
+from .hookspec import account_hookimpl, global_hookimpl  # noqa
+from . import eventlogger
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -74,3 +81,62 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
+
+
+def register_global_plugin(plugin):
+    """ Register a global plugin which implements one or more
+    of the :class:`deltachat.hookspec.Global` hooks.
+    """
+    gm = hookspec.Global._get_plugin_manager()
+    gm.register(plugin)
+    gm.check_pending()
+
+
+def unregister_global_plugin(plugin):
+    gm = hookspec.Global._get_plugin_manager()
+    gm.unregister(plugin)
+
+
+register_global_plugin(eventlogger)
+
+
+def run_cmdline(argv=None, account_plugins=None):
+    """ Run a simple default command line app, registering the specified
+    account plugins. """
+    import argparse
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
+    parser.add_argument("db", action="store", help="database file")
+    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
+    parser.add_argument("--email", action="store", help="email address")
+    parser.add_argument("--password", action="store", help="password")
+
+    args = parser.parse_args(argv[1:])
+
+    ac = Account(args.db)
+
+    if args.show_ffi:
+        log = eventlogger.FFIEventLogger(ac, "bot")
+        ac.add_account_plugin(log)
+
+    if not ac.is_configured():
+        assert args.email and args.password, (
+            "you must specify --email and --password once to configure this database/account"
+        )
+        ac.set_config("addr", args.email)
+        ac.set_config("mail_pw", args.password)
+        ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
+        ac.set_config("sentbox_watch", "0")
+
+    for plugin in account_plugins or []:
+        ac.add_account_plugin(plugin)
+
+    # start IO threads and configure if neccessary
+    ac.start()
+
+    print("{}: waiting for message".format(ac.get_config("addr")))
+
+    ac.wait_shutdown()

python/src/deltachat/account.py

@@ -2,22 +2,25 @@
 
 from __future__ import print_function
 import atexit
-import threading
-import re
-import time
+from contextlib import contextmanager
+from email.utils import parseaddr
+import queue
+from threading import Event
+import os
 from array import array
-try:
-    from queue import Queue, Empty
-except ImportError:
-    from Queue import Queue, Empty
-
 import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message
+from .message import Message, map_system_message
 from .contact import Contact
+from .tracker import ImexTracker
+from . import hookspec, iothreads
+
+
+class MissingCredentials(ValueError):
+    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
@@ -25,38 +28,53 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
+    MissingCredentials = MissingCredentials
+
+    def __init__(self, db_path, os_name=None):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        :param eventlogging: if False no eventlogging and no context callback will be configured
-        :param debug: turn on debug logging for events.
+        :param os_name: this will be put to the X-Mailer header in outgoing messages
         """
+        # initialize per-account plugin system
+        self._pm = hookspec.PerAccount._make_plugin_manager()
+        self.add_account_plugin(self)
+
         self._dc_context = ffi.gc(
-            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
-        if eventlogging:
-            self._evlogger = EventLogger(self._dc_context, logid, debug)
-            deltachat.set_context_callback(self._dc_context, self._process_event)
-            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
-        else:
-            self._threads = IOThreads(self._dc_context)
 
+        hook = hookspec.Global._get_plugin_manager().hook
+
+        self._threads = iothreads.IOThreads(self)
+        self._hook_event_queue = queue.Queue()
+        self._in_use_iter_events = False
+        self._shutdown_event = Event()
+
+        # open database
+        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
-        self._imex_events = Queue()
         atexit.register(self.shutdown)
+        hook.dc_account_init(account=self)
+
+    @hookspec.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        for name, kwargs in self._map_ffi_event(ffi_event):
+            ev = HookEvent(self, name=name, kwargs=kwargs)
+            self._hook_event_queue.put(ev)
 
     # def __del__(self):
     #    self.shutdown()
 
+    def ac_log_line(self, msg):
+        self._pm.hook.ac_log_line(message=msg)
+
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -94,9 +112,12 @@ class Account(object):
         """
         self._check_config_key(name)
         name = name.encode("utf8")
-        value = value.encode("utf8")
         if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
+        if value is not None:
+            value = value.encode("utf8")
+        else:
+            value = ffi.NULL
         lib.dc_set_config(self._dc_context, name, value)
 
     def get_config(self, name):
@@ -113,16 +134,27 @@ class Account(object):
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def configure(self, **kwargs):
-        """ set config values and configure this account.
+    def _preconfigure_keypair(self, addr, public, secret):
+        """See dc_preconfigure_keypair() in deltachat.h.
+
+        In other words, you don't need this.
+        """
+        res = lib.dc_preconfigure_keypair(self._dc_context,
+                                          as_dc_charpointer(addr),
+                                          as_dc_charpointer(public),
+                                          as_dc_charpointer(secret))
+        if res == 0:
+            raise Exception("Failed to set key")
+
+    def update_config(self, kwargs):
+        """ update config values.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for name, value in kwargs.items():
-            self.set_config(name, value)
-        lib.dc_configure(self._dc_context)
+        for key, value in kwargs.items():
+            self.set_config(key, str(value))
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -130,7 +162,19 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return lib.dc_is_configured(self._dc_context)
+        return bool(lib.dc_is_configured(self._dc_context))
+
+    def set_avatar(self, img_path):
+        """Set self avatar.
+
+        :raises ValueError: if profile image could not be set
+        :returns: None
+        """
+        if img_path is None:
+            self.set_config("selfavatar", None)
+        else:
+            assert os.path.exists(img_path), img_path
+            self.set_config("selfavatar", img_path)
 
     def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
@@ -148,11 +192,6 @@ class Account(object):
             raise ValueError("no flags set")
         lib.dc_empty_server(self._dc_context, flags)
 
-    def get_infostring(self):
-        """ return info of the configured account. """
-        self.check_is_configured()
-        return from_dc_charpointer(lib.dc_get_info(self._dc_context))
-
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -174,8 +213,7 @@ class Account(object):
 
         :returns: :class:`deltachat.contact.Contact`
         """
-        self.check_is_configured()
-        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
+        return Contact(self, const.DC_CONTACT_ID_SELF)
 
     def create_contact(self, email, name=None):
         """ create a (new) Contact. If there already is a Contact
@@ -186,11 +224,14 @@ class Account(object):
         :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        name = as_dc_charpointer(name)
-        email = as_dc_charpointer(email)
-        contact_id = lib.dc_create_contact(self._dc_context, name, email)
+        realname, addr = parseaddr(email)
+        if name:
+            realname = name
+        realname = as_dc_charpointer(realname)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
-        return Contact(self._dc_context, contact_id)
+        return Contact(self, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -203,6 +244,14 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
+    def get_contact_by_addr(self, email):
+        """ get a contact for the email address or None if it's blocked or doesn't exist. """
+        _, addr = parseaddr(email)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
+        if contact_id:
+            return self.get_contact_by_id(contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -222,7 +271,15 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+
+    def get_fresh_messages(self):
+        """ yield all fresh messages from all chats. """
+        dc_array = ffi.gc(
+            lib.dc_get_fresh_msgs(self._dc_context),
+            lib.dc_array_unref
+        )
+        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
     def create_chat_by_contact(self, contact):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
@@ -244,6 +301,9 @@ class Account(object):
         """ create or get an existing chat object for the
         the specified message.
 
+        If this message is in the deaddrop chat then
+        the sender will become an accepted contact.
+
         :param message: messsage id or message instance.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
@@ -296,6 +356,13 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -340,45 +407,37 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory. """
+        """ export public and private keys to the specified directory.
+
+        Note that the account does not have to be started.
+        """
         return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
+
+        Note that the account does not have to be started.
         """
         export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
-    def _imex_events_clear(self):
-        try:
-            while True:
-                self._imex_events.get_nowait()
-        except Empty:
-            pass
-
     def _export(self, path, imex_cmd):
-        self._imex_events_clear()
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
-        files_written = []
-        while True:
-            ev = self._imex_events.get()
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif isinstance(ev, bool):
-                if not ev:
-                    raise ValueError("export failed, exp-files: {}".format(files_written))
-                return files_written
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
+            return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
         """ Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
+
+        Note that the account does not have to be started.
         """
         self._import(path, imex_cmd=2)
 
@@ -391,12 +450,11 @@ class Account(object):
         self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
-        self._imex_events_clear()
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
-        if not self._imex_events.get():
-            raise ValueError("import from path '{}' failed".format(path))
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
+            imex_tracker.wait_finish()
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -461,63 +519,6 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def stop_ongoing(self):
-        lib.dc_stop_ongoing_process(self._dc_context)
-
-    #
-    # meta API for start/stop and event based processing
-    #
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.get_message_by_id(ev[2])
-
-    def start_threads(self, mvbox=False, sentbox=False):
-        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
-
-        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
-        :returns: None
-        """
-        if not self.is_configured():
-            self.configure()
-        self._threads.start(mvbox=mvbox, sentbox=sentbox)
-
-    def stop_threads(self, wait=True):
-        """ stop IMAP/SMTP threads. """
-        if self._threads.is_started():
-            self.stop_ongoing()
-            self._threads.stop(wait=wait)
-
-    def shutdown(self, wait=True):
-        """ stop threads and close and remove underlying dc_context and callbacks. """
-        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
-            # print("SHUTDOWN", self)
-            self.stop_threads(wait=False)
-            lib.dc_close(self._dc_context)
-            self.stop_threads(wait=wait)  # to wait for threads
-            deltachat.clear_context_callback(self._dc_context)
-            del self._dc_context
-            atexit.unregister(self.shutdown)
-
-    def _process_event(self, ctx, evt_name, data1, data2):
-        assert ctx == self._dc_context
-        if hasattr(self, "_evlogger"):
-            self._evlogger(evt_name, data1, data2)
-            method = getattr(self, "on_" + evt_name.lower(), None)
-            if method is not None:
-                method(data1, data2)
-        return 0
-
-    def on_dc_event_imex_progress(self, data1, data2):
-        if data1 == 1000:
-            self._imex_events.put(True)
-        elif data1 == 0:
-            self._imex_events.put(False)
-
-    def on_dc_event_imex_file_written(self, data1, data2):
-        self._imex_events.put(data1)
-
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
@@ -532,149 +533,122 @@ class Account(object):
         if dc_res == 0:
             raise ValueError("no chat is streaming locations")
 
+    #
+    # meta API for start/stop and event based processing
+    #
 
-class IOThreads:
-    def __init__(self, dc_context, log_event=lambda *args: None):
-        self._dc_context = dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-        self._log_event = log_event
+    def add_account_plugin(self, plugin, name=None):
+        """ add an account plugin which implements one or more of
+        the :class:`deltachat.hookspec.PerAccount` hooks.
+        """
+        self._pm.register(plugin, name=name)
+        self._pm.check_pending()
+        return plugin
 
-    def is_started(self):
-        return len(self._name2thread) > 0
+    @contextmanager
+    def temp_plugin(self, plugin):
+        """ run a with-block with the given plugin temporarily registered. """
+        self._pm.register(plugin)
+        yield plugin
+        self._pm.unregister(plugin)
 
-    def start(self, imap=True, smtp=True, mvbox=False, sentbox=False):
-        assert not self.is_started()
-        if imap:
-            self._start_one_thread("inbox", self.imap_thread_run)
-        if mvbox:
-            self._start_one_thread("mvbox", self.mvbox_thread_run)
-        if sentbox:
-            self._start_one_thread("sentbox", self.sentbox_thread_run)
-        if smtp:
-            self._start_one_thread("smtp", self.smtp_thread_run)
+    def stop_ongoing(self):
+        """ Stop ongoing securejoin, configuration or other core jobs. """
+        lib.dc_stop_ongoing_process(self._dc_context)
 
-    def _start_one_thread(self, name, func):
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        t.setDaemon(1)
-        t.start()
+    def start(self, callback_thread=True):
+        """ start this account (activate imap/smtp threads etc.)
+        and return immediately.
 
-    def stop(self, wait=False):
-        self._thread_quitflag = True
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        lib.dc_interrupt_mvbox_idle(self._dc_context)
-        lib.dc_interrupt_sentbox_idle(self._dc_context)
-        if wait:
-            for name, thread in self._name2thread.items():
-                thread.join()
+        If this account is not configured, an internal configuration
+        job will be scheduled if config values are sufficiently specified.
 
-    def imap_thread_run(self):
-        self._log_event("py-bindings-info", 0, "INBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_imap_jobs(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_imap_fetch(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_imap_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
+        You may call `wait_shutdown` or `shutdown` after the
+        account is in started mode.
 
-    def mvbox_thread_run(self):
-        self._log_event("py-bindings-info", 0, "MVBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_mvbox_jobs(self._dc_context)
-            lib.dc_perform_mvbox_fetch(self._dc_context)
-            lib.dc_perform_mvbox_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "MVBOX THREAD FINISHED")
+        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
 
-    def sentbox_thread_run(self):
-        self._log_event("py-bindings-info", 0, "SENTBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_sentbox_jobs(self._dc_context)
-            lib.dc_perform_sentbox_fetch(self._dc_context)
-            lib.dc_perform_sentbox_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "SENTBOX THREAD FINISHED")
+        :returns: None
+        """
+        if not self.is_configured():
+            if not self.get_config("addr") or not self.get_config("mail_pw"):
+                raise MissingCredentials("addr or mail_pwd not set in config")
+            lib.dc_configure(self._dc_context)
+        self._threads.start(callback_thread=callback_thread)
 
-    def smtp_thread_run(self):
-        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_smtp_jobs(self._dc_context)
-            lib.dc_perform_smtp_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
+    def wait_shutdown(self):
+        """ wait until shutdown of this account has completed. """
+        self._shutdown_event.wait()
 
+    def shutdown(self, wait=True):
+        """ shutdown account, stop threads and close and remove
+        underlying dc_context and callbacks. """
+        dc_context = self._dc_context
+        if dc_context is None:
+            return
 
-class EventLogger:
-    _loglock = threading.RLock()
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=False)
+        lib.dc_close(dc_context)
+        self._hook_event_queue.put(None)
+        self._threads.stop(wait=wait)  # to wait for threads
+        self._dc_context = None
+        atexit.unregister(self.shutdown)
+        self._shutdown_event.set()
+        hook = hookspec.Global._get_plugin_manager().hook
+        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
 
-    def __init__(self, dc_context, logid=None, debug=True):
-        self._dc_context = dc_context
-        self._event_queue = Queue()
-        self._debug = debug
-        if logid is None:
-            logid = str(self._dc_context).strip(">").split()[-1]
-        self.logid = logid
-        self._timeout = None
-        self.init_time = time.time()
-
-    def __call__(self, evt_name, data1, data2):
-        self._log_event(evt_name, data1, data2)
-        self._event_queue.put((evt_name, data1, data2))
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get()
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout or self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev[0] == "DC_EVENT_ERROR":
-            raise ValueError("{}({!r},{!r})".format(*ev))
-        return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
+    def _handle_current_events(self):
+        """ handle all currently queued events and then return. """
         while 1:
             try:
-                ev = self._event_queue.get(False)
-            except Empty:
+                event = self._hook_event_queue.get(block=False)
+            except queue.Empty:
                 break
             else:
-                assert not rex.match(ev[0]), "event found {}".format(ev)
+                event.call_hook()
 
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
-        rex = re.compile("(?:{}).*".format(event_name_regex))
+    def iter_events(self, timeout=None):
+        """ yield hook events until shutdown.
+
+        It is not allowed to call iter_events() from multiple threads.
+        """
+        if self._in_use_iter_events:
+            raise RuntimeError("can only call iter_events() from one thread")
+        self._in_use_iter_events = True
         while 1:
-            ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev[0]):
-                return ev
+            event = self._hook_event_queue.get(timeout=timeout)
+            if event is None:
+                break
+            yield event
 
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev[2]):
-                return ev
-
-    def _log_event(self, evt_name, data1, data2):
-        # don't show events that are anyway empty impls now
-        if evt_name == "DC_EVENT_GET_STRING":
-            return
-        if self._debug:
-            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
-            self._log(evpart)
-
-    def _log(self, msg):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        with self._loglock:
-            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
+    def _map_ffi_event(self, ffi_event):
+        name = ffi_event.name
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+            data1 = ffi_event.data1
+            if data1 == 0 or data1 == 1000:
+                success = data1 == 1000
+                yield "ac_configure_completed", dict(success=success)
+        elif name == "DC_EVENT_INCOMING_MSG":
+            msg = self.get_message_by_id(ffi_event.data2)
+            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+        elif name == "DC_EVENT_MSGS_CHANGED":
+            if ffi_event.data2 != 0:
+                msg = self.get_message_by_id(ffi_event.data2)
+                if msg.is_outgoing():
+                    res = map_system_message(msg)
+                    if res and res[0].startswith("ac_member"):
+                        yield res
+                    yield "ac_outgoing_message", dict(message=msg)
+                elif msg.is_in_fresh():
+                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+        elif name == "DC_EVENT_MSG_DELIVERED":
+            msg = self.get_message_by_id(ffi_event.data2)
+            yield "ac_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_CHAT_MODIFIED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            yield "ac_chat_modified", dict(chat=chat)
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
@@ -701,3 +675,17 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
+
+
+class HookEvent:
+    def __init__(self, account, name, kwargs):
+        assert hasattr(account._pm.hook, name), name
+        self.account = account
+        self.name = name
+        self.kwargs = kwargs
+
+    def call_hook(self):
+        hook = getattr(self.account._pm.hook, self.name, None)
+        if hook is None:
+            raise ValueError("event_name {} unknown".format(self.name))
+        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -2,6 +2,7 @@
 
 import mimetypes
 import calendar
+import json
 from datetime import datetime
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
@@ -29,7 +30,7 @@ class Chat(object):
         return not (self == other)
 
     def __repr__(self):
-        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
+        return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
     def _dc_chat(self):
@@ -50,6 +51,16 @@ class Chat(object):
 
     # ------  chat status/metadata API ------------------------------
 
+    def is_group(self):
+        """ return true if this chat is a group chat.
+
+        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
+        """
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
+
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
 
@@ -57,6 +68,13 @@ class Chat(object):
         """
         return self.id == const.DC_CHAT_ID_DEADDROP
 
+    def is_muted(self):
+        """ return true if this chat is muted.
+
+        :returns: True if chat is muted, False otherwise.
+        """
+        return lib.dc_chat_is_muted(self._dc_chat)
+
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -83,14 +101,45 @@ class Chat(object):
     def set_name(self, name):
         """ set name of this chat.
 
-        :param: name as a unicode string.
+        :param name: as a unicode string.
         :returns: None
         """
         name = as_dc_charpointer(name)
         return lib.dc_set_chat_name(self._dc_context, self.id, name)
 
+    def mute(self, duration=None):
+        """ mutes the chat
+
+        :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
+        :returns: None
+        """
+        if duration is None:
+            mute_duration = -1
+        else:
+            mute_duration = duration
+        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, mute_duration)
+        if not bool(ret):
+            raise ValueError("Call to dc_set_chat_mute_duration failed")
+
+    def unmute(self):
+        """ unmutes the chat
+
+        :returns: None
+        """
+        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, 0)
+        if not bool(ret):
+            raise ValueError("Failed to unmute chat")
+
+    def get_mute_duration(self):
+        """ Returns the number of seconds until the mute of this chat is lifted.
+
+        :param duration:
+        :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
+        """
+        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
+
     def get_type(self):
-        """ return type of this chat.
+        """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -108,6 +157,30 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
+    def send_msg(self, msg):
+        """send a message by using a ready Message object.
+
+        :param msg: a :class:`deltachat.message.Message` instance
+           previously returned by
+           e.g. :meth:`deltachat.message.Message.new_empty` or
+           :meth:`prepare_file`.
+        :raises ValueError: if message can not be sent.
+
+        :returns: a :class:`deltachat.message.Message` instance as
+           sent out.  This is the same object as was passed in, which
+           has been modified with the new state of the core.
+        """
+        if msg.is_out_preparing():
+            assert msg.id != 0
+            # get a fresh copy of dc_msg, the core needs it
+            msg = Message.from_db(self.account, msg.id)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        # modify message in place to avoid bad state for the caller
+        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
+        return msg
+
     def send_text(self, text):
         """ send a text message and return the resulting Message instance.
 
@@ -129,9 +202,12 @@ class Chat(object):
         :raises ValueError: if message can not be send/chat does not exist.
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
-        msg = self.prepare_message_file(path=path, mime_type=mime_type)
-        self.send_prepared(msg)
-        return msg
+        msg = Message.new_empty(self.account, view_type="file")
+        msg.set_file(path, mime_type)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        return Message.from_db(self.account, sent_id)
 
     def send_image(self, path):
         """ send an image message and return the resulting Message instance.
@@ -141,9 +217,12 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         mime_type = mimetypes.guess_type(path)[0]
-        msg = self.prepare_message_file(path=path, mime_type=mime_type, view_type="image")
-        self.send_prepared(msg)
-        return msg
+        msg = Message.new_empty(self.account, view_type="image")
+        msg.set_file(path, mime_type)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
         """ create a new prepared message.
@@ -242,6 +321,12 @@ class Chat(object):
         """
         return lib.dc_marknoticed_chat(self._dc_context, self.id)
 
+    def get_summary(self):
+        """ return dictionary with summary information. """
+        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
+        s = from_dc_charpointer(dc_res)
+        return json.loads(s)
+
     # ------  group management API ------------------------------
 
     def add_contact(self, contact):
@@ -278,7 +363,7 @@ class Chat(object):
             lib.dc_array_unref
         )
         return list(iter_array(
-            dc_array, lambda id: Contact(self._dc_context, id))
+            dc_array, lambda id: Contact(self.account, id))
         )
 
     def set_profile_image(self, img_path):
@@ -324,6 +409,12 @@ class Chat(object):
             return None
         return from_dc_charpointer(dc_res)
 
+    def get_color(self):
+        """return the color of the chat.
+        :returns: color as 0x00rrggbb
+        """
+        return lib.dc_chat_get_color(self._dc_chat)
+
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -332,6 +423,12 @@ class Chat(object):
         """
         return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
 
+    def is_archived(self):
+        """return True if this chat is archived.
+        :returns: True if archived.
+        """
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.
 

python/src/deltachat/const.py

@@ -11,6 +11,7 @@ from os.path import join as joinpath
 DC_GCL_ARCHIVED_ONLY = 0x01
 DC_GCL_NO_SPECIALS = 0x02
 DC_GCL_ADD_ALLDONE_HINT = 0x04
+DC_GCL_FOR_FORWARDING = 0x08
 DC_GCL_VERIFIED_ONLY = 0x01
 DC_GCL_ADD_SELF = 0x02
 DC_QR_ASK_VERIFYCONTACT = 200
@@ -18,6 +19,7 @@ DC_QR_ASK_VERIFYGROUP = 202
 DC_QR_FPR_OK = 210
 DC_QR_FPR_MISMATCH = 220
 DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ACCOUNT = 250
 DC_QR_ADDR = 320
 DC_QR_TEXT = 330
 DC_QR_URL = 332
@@ -68,7 +70,6 @@ DC_LP_SMTP_SOCKET_SSL = 0x20000
 DC_LP_SMTP_SOCKET_PLAIN = 0x40000
 DC_CERTCK_AUTO = 0
 DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_HOSTNAMES = 2
 DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
 DC_EMPTY_MVBOX = 0x01
 DC_EMPTY_INBOX = 0x02
@@ -98,19 +99,22 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091
 DC_STR_SELFNOTINGRP = 21
+DC_KEY_GEN_DEFAULT = 0
+DC_KEY_GEN_RSA2048 = 1
+DC_KEY_GEN_ED25519 = 2
 DC_PROVIDER_STATUS_OK = 1
 DC_PROVIDER_STATUS_PREPARATION = 2
 DC_PROVIDER_STATUS_BROKEN = 3
+DC_CHAT_VISIBILITY_NORMAL = 0
+DC_CHAT_VISIBILITY_ARCHIVED = 1
+DC_CHAT_VISIBILITY_PINNED = 2
 DC_STR_NOMESSAGES = 1
 DC_STR_SELF = 2
 DC_STR_DRAFT = 3
-DC_STR_MEMBER = 4
-DC_STR_CONTACT = 6
 DC_STR_VOICEMESSAGE = 7
 DC_STR_DEADDROP = 8
 DC_STR_IMAGE = 9
@@ -142,7 +146,6 @@ DC_STR_ARCHIVEDCHATS = 40
 DC_STR_STARREDMSGS = 41
 DC_STR_AC_SETUP_MSG_SUBJECT = 42
 DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_SELFTALK_SUBTITLE = 50
 DC_STR_CANNOT_LOGIN = 60
 DC_STR_SERVER_RESPONSE = 61
 DC_STR_MSGACTIONBYUSER = 62
@@ -158,7 +161,7 @@ DC_STR_COUNT = 68
 
 def read_event_defines(f):
     rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/contact.py

@@ -10,8 +10,9 @@ class Contact(object):
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
-    def __init__(self, dc_context, id):
-        self._dc_context = dc_context
+    def __init__(self, account, id):
+        self.account = account
+        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
@@ -47,3 +48,17 @@ class Contact(object):
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
+
+    def get_profile_image(self):
+        """Get contact profile image.
+
+        :returns: path to profile image, None if no profile image exists.
+        """
+        dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)
+
+    def get_chat(self):
+        """return 1:1 chat for this contact. """
+        return self.account.create_chat_by_contact(self)

python/src/deltachat/eventlogger.py

@@ -0,0 +1,137 @@
+import deltachat
+import threading
+import time
+import re
+from queue import Queue, Empty
+from .hookspec import account_hookimpl, global_hookimpl
+
+
+@global_hookimpl
+def dc_account_init(account):
+    # send all FFI events for this account to a plugin hook
+    def _ll_event(ctx, evt_name, data1, data2):
+        assert ctx == account._dc_context
+        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+        account._pm.hook.ac_process_ffi_event(
+            account=account, ffi_event=ffi_event
+        )
+    deltachat.set_context_callback(account._dc_context, _ll_event)
+
+
+@global_hookimpl
+def dc_account_after_shutdown(dc_context):
+    deltachat.clear_context_callback(dc_context)
+
+
+class FFIEvent:
+    def __init__(self, name, data1, data2):
+        self.name = name
+        self.data1 = data1
+        self.data2 = data2
+
+    def __str__(self):
+        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
+
+
+class FFIEventLogger:
+    """ If you register an instance of this logger with an Account
+    you'll get all ffi-events printed.
+    """
+    # to prevent garbled logging
+    _loglock = threading.RLock()
+
+    def __init__(self, account, logid):
+        """
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        """
+        self.account = account
+        self.logid = logid
+        self.init_time = time.time()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._log_event(ffi_event)
+
+    def _log_event(self, ffi_event):
+        # don't show events that are anyway empty impls now
+        if ffi_event.name == "DC_EVENT_GET_STRING":
+            return
+        self.account.ac_log_line(str(ffi_event))
+
+    @account_hookimpl
+    def ac_log_line(self, message):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        elapsed = time.time() - self.init_time
+        locname = tname
+        if self.logid:
+            locname += "-" + self.logid
+        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        with self._loglock:
+            print(s, flush=True)
+
+
+class FFIEventTracker:
+    def __init__(self, account, timeout=None):
+        self.account = account
+        self._timeout = timeout
+        self._event_queue = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._event_queue.put(ffi_event)
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get(check_error=check_error)
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout if timeout is not None else self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev.name == "DC_EVENT_ERROR":
+            raise ValueError(str(ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev.name), "event found {}".format(ev)
+
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            ev = self.get(timeout=timeout, check_error=check_error)
+            if rex.match(ev.name):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev.data2):
+                return ev
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.account.get_message_by_id(ev.data2)
+
+    def wait_next_messages_changed(self):
+        """ wait for and return next message-changed message or None
+        if the event contains no msgid"""
+        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
+        if ev.data2 > 0:
+            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/hookspec.py

@@ -0,0 +1,92 @@
+""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
+
+import pluggy
+
+
+account_spec_name = "deltachat-account"
+account_hookspec = pluggy.HookspecMarker(account_spec_name)
+account_hookimpl = pluggy.HookimplMarker(account_spec_name)
+
+global_spec_name = "deltachat-global"
+global_hookspec = pluggy.HookspecMarker(global_spec_name)
+global_hookimpl = pluggy.HookimplMarker(global_spec_name)
+
+
+class PerAccount:
+    """ per-Account-instance hook specifications.
+
+    Except for ac_process_ffi_event all hooks are executed
+    in the thread which calls Account.wait_shutdown().
+    """
+    @classmethod
+    def _make_plugin_manager(cls):
+        pm = pluggy.PluginManager(account_spec_name)
+        pm.add_hookspecs(cls)
+        return pm
+
+    @account_hookspec
+    def ac_process_ffi_event(self, ffi_event):
+        """ process a CFFI low level events for a given account.
+
+        ffi_event has "name", "data1", "data2" values as specified
+        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
+        """
+
+    @account_hookspec
+    def ac_log_line(self, message):
+        """ log a message related to the account. """
+
+    @account_hookspec
+    def ac_configure_completed(self, success):
+        """ Called when a configure process completed. """
+
+    @account_hookspec
+    def ac_incoming_message(self, message):
+        """ Called on any incoming message (to deaddrop or chat). """
+
+    @account_hookspec
+    def ac_outgoing_message(self, message):
+        """ Called on each outgoing message (both system and "normal")."""
+
+    @account_hookspec
+    def ac_message_delivered(self, message):
+        """ Called when an outgoing message has been delivered to SMTP. """
+
+    @account_hookspec
+    def ac_chat_modified(self, chat):
+        """ Chat was created or modified regarding membership, avatar, title. """
+
+    @account_hookspec
+    def ac_member_added(self, chat, contact, message):
+        """ Called for each contact added to an accepted chat. """
+
+    @account_hookspec
+    def ac_member_removed(self, chat, contact, message):
+        """ Called for each contact removed from a chat.  """
+
+
+class Global:
+    """ global hook specifications using a per-process singleton
+    plugin manager instance.
+
+    """
+    _plugin_manager = None
+
+    @classmethod
+    def _get_plugin_manager(cls):
+        if cls._plugin_manager is None:
+            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
+            pm.add_hookspecs(cls)
+        return cls._plugin_manager
+
+    @global_hookspec
+    def dc_account_init(self, account):
+        """ called when `Account::__init__()` function starts executing. """
+
+    @global_hookspec
+    def dc_account_after_shutdown(self, account, dc_context):
+        """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -0,0 +1,106 @@
+
+import threading
+import time
+
+from contextlib import contextmanager
+
+from .capi import lib
+
+
+class IOThreads:
+    def __init__(self, account):
+        self.account = account
+        self._dc_context = account._dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+
+    def is_started(self):
+        return len(self._name2thread) > 0
+
+    def start(self, callback_thread):
+        assert not self.is_started()
+        self._start_one_thread("inbox", self.imap_thread_run)
+        self._start_one_thread("smtp", self.smtp_thread_run)
+
+        if callback_thread:
+            self._start_one_thread("cb", self.cb_thread_run)
+
+        if int(self.account.get_config("mvbox_watch")):
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+
+        if int(self.account.get_config("sentbox_watch")):
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
+
+    @contextmanager
+    def log_execution(self, message):
+        self.account.ac_log_line(message + " START")
+        yield
+        self.account.ac_log_line(message + " FINISHED")
+
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+
+        # Workaround for a race condition. Make sure that thread is
+        # not in between checking for quitflag and entering idle.
+        time.sleep(0.5)
+
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        if "mvbox" in self._name2thread:
+            lib.dc_interrupt_mvbox_idle(self._dc_context)
+        if "sentbox" in self._name2thread:
+            lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                if thread != threading.currentThread():
+                    thread.join()
+
+    def cb_thread_run(self):
+        with self.log_execution("CALLBACK THREAD START"):
+            it = self.account.iter_events()
+            while not self._thread_quitflag:
+                try:
+                    ev = next(it)
+                except StopIteration:
+                    break
+                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
+                ev.call_hook()
+
+    def imap_thread_run(self):
+        with self.log_execution("INBOX THREAD START"):
+            while not self._thread_quitflag:
+                lib.dc_perform_imap_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_idle(self._dc_context)
+
+    def mvbox_thread_run(self):
+        with self.log_execution("MVBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_mvbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_idle(self._dc_context)
+
+    def sentbox_thread_run(self):
+        with self.log_execution("SENTBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_sentbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_idle(self._dc_context)
+
+    def smtp_thread_run(self):
+        with self.log_execution("SMTP THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_smtp_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -1,7 +1,6 @@
 """ The Message object. """
 
 import os
-import shutil
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -29,7 +28,9 @@ class Message(object):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
+        c = self.get_sender_contact()
+        return "<Message id={} sender={}/{} outgoing={} chat={}/{}>".format(
+            self.id, c.id, c.addr, self.is_outgoing(), self.chat.id, self.chat.get_name())
 
     @classmethod
     def from_db(cls, account, id):
@@ -51,6 +52,16 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
+    def accept_sender_contact(self):
+        """ ensure that the sender is an accepted contact
+        and that the message has a non-deaddrop chat object.
+        """
+        self.account.create_chat_by_message(self)
+        self._dc_msg = ffi.gc(
+                lib.dc_get_msg(self._dc_context, self.id),
+                lib.dc_msg_unref
+        )
+
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -58,8 +69,6 @@ class Message(object):
 
     def set_text(self, text):
         """set text of this message. """
-        assert self.id > 0, "message not prepared"
-        assert self.is_out_preparing()
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
@@ -72,19 +81,6 @@ class Message(object):
         mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type)
         if not os.path.exists(path):
             raise ValueError("path does not exist: {!r}".format(path))
-        blobdir = self.account.get_blobdir()
-        if not path.startswith(blobdir):
-            for i in range(50):
-                ext = "" if i == 0 else "-" + str(i)
-                dest = os.path.join(blobdir, os.path.basename(path) + ext)
-                if os.path.exists(dest):
-                    continue
-                shutil.copyfile(path, dest)
-                break
-            else:
-                raise ValueError("could not create blobdir-path for {}".format(path))
-            path = dest
-        assert path.startswith(blobdir), path
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
@@ -97,6 +93,10 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
+    def is_system_message(self):
+        """ return True if this message is a system/info message. """
+        return lib.dc_msg_is_info(self._dc_msg)
+
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
@@ -175,6 +175,13 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
+    def get_sender_chat(self):
+        """return the 1:1 chat with the sender of this message.
+
+        :returns: :class:`deltachat.chat.Chat` instance
+        """
+        return self.get_sender_contact().get_chat()
+
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
@@ -182,7 +189,7 @@ class Message(object):
         """
         from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self._dc_context, contact_id)
+        return Contact(self.account, contact_id)
 
     #
     # Message State query methods
@@ -190,7 +197,7 @@ class Message(object):
     @property
     def _msgstate(self):
         if self.id == 0:
-            dc_msg = self.message._dc_msg
+            dc_msg = self._dc_msg
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
@@ -223,6 +230,13 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
+    def is_outgoing(self):
+        """Return True if Message is outgoing. """
+        return self._msgstate in (
+            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
+            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
+            const.DC_STATE_OUT_DELIVERED)
+
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -285,6 +299,10 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
+    def mark_seen(self):
+        """ mark this message as seen. """
+        self.account.mark_seen_messages([self.id])
+
 
 # some code for handling DC_MSG_* view types
 
@@ -304,3 +322,29 @@ def get_viewtype_code_from_name(view_type_name):
             return code
     raise ValueError("message typecode not found for {!r}, "
                      "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
+
+
+#
+# some helper code for turning system messages into hook events
+#
+
+def map_system_message(msg):
+    if msg.is_system_message():
+        res = parse_system_add_remove(msg.text)
+        if res:
+            contact = msg.account.get_contact_by_addr(res[1])
+            if contact:
+                d = dict(chat=msg.chat, contact=contact, message=msg)
+                return "ac_member_" + res[0], d
+
+
+def parse_system_add_remove(text):
+    # Member Me (x@y) removed by a@b.
+    # Member x@y removed by a@b
+    text = text.lower()
+    parts = text.split()
+    if parts[0] == "member":
+        if parts[2] in ("removed", "added"):
+            return parts[2], parts[1]
+        if parts[3] in ("removed", "added"):
+            return parts[3], parts[2].strip("()")

python/src/deltachat/provider.py

@@ -11,27 +11,18 @@ class ProviderNotFoundError(Exception):
 class Provider(object):
     """Provider information.
 
-    :param domain: The domain to get the provider info for, this is
-    normally the part following the `@` of the domain.
+    :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, domain):
+    def __init__(self, account, addr):
         provider = ffi.gc(
-            lib.dc_provider_new_from_domain(as_dc_charpointer(domain)),
+            lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
         )
         if provider == ffi.NULL:
             raise ProviderNotFoundError("Provider not found")
         self._provider = provider
 
-    @classmethod
-    def from_email(cls, email):
-        """Create provider info from an email address.
-
-        :param email: Email address to get provider info for.
-        """
-        return cls(email.split('@')[-1])
-
     @property
     def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
@@ -39,21 +30,10 @@ class Provider(object):
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def name(self):
-        """The name of the provider."""
-        return from_dc_charpointer(lib.dc_provider_get_name(self._provider))
-
-    @property
-    def markdown(self):
-        """Content of the information page, formatted as markdown."""
+    def get_before_login_hints(self):
+        """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_markdown(self._provider))
-
-    @property
-    def status_date(self):
-        """The date the provider info was last updated, as a string."""
-        return from_dc_charpointer(
-            lib.dc_provider_get_status_date(self._provider))
+            lib.dc_provider_get_before_login_hints(self._provider))
 
     @property
     def status(self):

python/src/deltachat/testplugin.py

@@ -0,0 +1,381 @@
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import queue
+import threading
+import fnmatch
+import pytest
+import requests
+import time
+from . import Account, const
+from .tracker import ConfigureTracker
+from .capi import lib
+from .eventlogger import FFIEventLogger, FFIEventTracker
+from _pytest.monkeypatch import MonkeyPatch
+from _pytest._code import Source
+
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+    parser.addoption(
+        "--ignored", action="store_true",
+        help="Also run tests marked with the ignored marker",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
+    )
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_runtest_setup(item):
+    if (list(item.iter_markers(name="ignored"))
+            and not item.config.getoption("ignored")):
+        pytest.skip("Ignored tests not requested, use --ignored")
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    m = MonkeyPatch()
+    try:
+        m.setattr(sys.stdout, "write", lambda x: len(x))
+        ac = Account(t)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        m.undo()
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "?" in cfg:
+            url, token = cfg.split("?", 1)
+            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url):
+        self.configlist = []
+        self.url = url
+
+    def get(self, index):
+        try:
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture
+def data(request):
+    class Data:
+        def __init__(self):
+            # trying to find test data heuristically
+            # because we are run from a dev-setup with pytest direct,
+            # through tox, and then maybe also from deltachat-binding
+            # users like "deltabot".
+            self.paths = [os.path.normpath(x) for x in [
+                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
+                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
+            ]]
+
+        def get_path(self, bn):
+            """ return path of file or None if it doesn't exist. """
+            for path in self.paths:
+                fn = os.path.join(path, *bn.split("/"))
+                if os.path.exists(fn):
+                    return fn
+            print("WARNING: path does not exist: {!r}".format(fn))
+
+        def read_path(self, bn, mode="r"):
+            fn = self.get_path(bn)
+            if fn is not None:
+                with open(fn, mode) as f:
+                    return f.read()
+
+    return Data()
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
+                                    "dom", "elena", "fiona"]
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid, quiet=False):
+            ac = Account(path)
+            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
+            if not quiet:
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
+
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
+                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
+                if fname_pub and fname_sec:
+                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                    return True
+                else:
+                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def get_online_config(self, pre_generated_key=True, quiet=False):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+                                           pre_generated_key=True, quiet=False, config={}):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key, quiet=quiet)
+            configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
+            configdict["mvbox_move"] = str(int(move))
+            configdict["sentbox_watch"] = str(int(sentbox))
+            ac.update_config(configdict)
+            ac.start()
+            return ac
+
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+            ac1._configtracker.wait_imap_connected()
+            ac1._configtracker.wait_smtp_connected()
+            ac1._configtracker.wait_finish()
+            return ac1
+
+        def get_two_online_accounts(self, move=False, quiet=False):
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac2 = self.get_online_configuring_account(quiet=quiet)
+            ac1._configtracker.wait_finish()
+            ac2._configtracker.wait_finish()
+            return ac1, ac2
+
+        def clone_online_account(self, account, pre_generated_key=True):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            ac.update_config(dict(
+                addr=account.get_config("addr"),
+                mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
+                mvbox_move=account.get_config("mvbox_move"),
+                sentbox_watch=account.get_config("sentbox_watch"),
+            ))
+            ac.start()
+            return ac
+
+        def run_bot_process(self, module, ffi=True):
+            fn = module.__file__
+
+            bot_ac, bot_cfg = self.get_online_config()
+
+            args = [
+                sys.executable,
+                "-u",
+                fn,
+                "--email", bot_cfg["addr"],
+                "--password", bot_cfg["mail_pw"],
+                bot_ac.db_path,
+            ]
+            if ffi:
+                args.insert(-1, "--show-ffi")
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=0,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
+            )
+            bot = BotProcess(popen, bot_cfg)
+            self._finalizers.append(bot.kill)
+            return bot
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+class BotProcess:
+    def __init__(self, popen, bot_cfg):
+        self.popen = popen
+        self.addr = bot_cfg["addr"]
+
+        # we read stdout as quickly as we can in a thread and make
+        # the (unicode) lines available for readers through a queue.
+        self.stdout_queue = queue.Queue()
+        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
+        t.setDaemon(1)
+        t.start()
+
+    def _run_stdout_thread(self):
+        try:
+            while 1:
+                line = self.popen.stdout.readline()
+                if not line:
+                    break
+                line = line.strip()
+                self.stdout_queue.put(line)
+        finally:
+            self.stdout_queue.put(None)
+
+    def kill(self):
+        self.popen.kill()
+
+    def wait(self, timeout=30):
+        self.popen.wait(timeout=timeout)
+
+    def fnmatch_lines(self, pattern_lines):
+        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
+        for next_pattern in patterns:
+            print("+++FNMATCH:", next_pattern)
+            ignored = []
+            while 1:
+                line = self.stdout_queue.get(timeout=15)
+                if line is None:
+                    if ignored:
+                        print("BOT stdout terminated after these lines")
+                        for line in ignored:
+                            print(line)
+                    raise IOError("BOT stdout-thread terminated")
+                if fnmatch.fnmatch(line, next_pattern):
+                    print("+++MATCHED:", line)
+                    break
+                else:
+                    print("+++IGN:", line)
+                    ignored.append(line)
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()

python/src/deltachat/tracker.py

@@ -0,0 +1,76 @@
+
+from queue import Queue
+from threading import Event
+
+from .hookspec import account_hookimpl
+
+
+class ImexFailed(RuntimeError):
+    """ Exception for signalling that import/export operations failed."""
+
+
+class ImexTracker:
+    def __init__(self):
+        self._imex_events = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
+            self._imex_events.put(ffi_event.data1)
+        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
+            self._imex_events.put(ffi_event.data1)
+
+    def wait_finish(self, progress_timeout=60):
+        """ Return list of written files, raise ValueError if ExportFailed. """
+        files_written = []
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif ev == 0:
+                raise ImexFailed("export failed, exp-files: {}".format(files_written))
+            elif ev == 1000:
+                return files_written
+
+
+class ConfigureFailed(RuntimeError):
+    """ Exception for signalling that configuration failed."""
+
+
+class ConfigureTracker:
+    ConfigureFailed = ConfigureFailed
+
+    def __init__(self):
+        self._configure_events = Queue()
+        self._smtp_finished = Event()
+        self._imap_finished = Event()
+        self._ffi_events = []
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._ffi_events.append(ffi_event)
+        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
+            self._smtp_finished.set()
+        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
+            self._imap_finished.set()
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        self._configure_events.put(success)
+
+    def wait_smtp_connected(self):
+        """ wait until smtp is configured. """
+        self._smtp_finished.wait()
+
+    def wait_imap_connected(self):
+        """ wait until smtp is configured. """
+        self._imap_finished.wait()
+
+    def wait_finish(self):
+        """ wait until configure is completed.
+
+        Raise Exception if Configure failed
+        """
+        if not self._configure_events.get():
+            content = "\n".join(map(str, self._ffi_events))
+            raise ConfigureFailed(content)

python/tests/conftest.py

@@ -1,268 +1,18 @@
 from __future__ import print_function
-import os
-import pytest
-import requests
-import time
-from deltachat import Account
-from deltachat import const
-from deltachat.capi import lib
-import tempfile
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-
-
-def pytest_configure(config):
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_PY_LIVECONFIG')
-        if cfg:
-            config.option.liveconfig = cfg
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    try:
-        ac = Account(t, eventlogging=False)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "#" in cfg:
-            url, token = cfg.split("#", 1)
-            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-@pytest.fixture(scope="session")
-def data():
-    class Data:
-        def __init__(self):
-            self.path = os.path.join(os.path.dirname(__file__), "data")
-
-        def get_path(self, bn):
-            fn = os.path.join(self.path, bn)
-            assert os.path.exists(fn)
-            return fn
-    return Data()
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url, create_token):
-        self.configlist = []
-        for i in range(2):
-            res = requests.post(url, json={"token_create_user": int(create_token)})
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            url, create_token = liveconfig_opt.split("#", 1)
-            return SessionLiveConfigFromURL(url, create_token)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self.init_time = time.time()
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-        def make_account(self, path, logid):
-            ac = Account(path, logid=logid)
-            self._finalizers.append(ac.shutdown)
-            return ac
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(2)
-            return ac
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def peek_online_config(self):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
-            return session_liveconfig.get(self.live_count)
-
-        def get_online_config(self):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            return ac, dict(configdict)
-
-        def get_online_configuring_account(self, mvbox=False, sentbox=False):
-            ac, configdict = self.get_online_config()
-            ac.configure(**configdict)
-            ac.start_threads(mvbox=mvbox, sentbox=sentbox)
-            return ac
-
-        def get_one_online_account(self):
-            ac1 = self.get_online_configuring_account()
-            wait_successful_IMAP_SMTP_connection(ac1)
-            wait_configuration_progress(ac1, 1000)
-            return ac1
-
-        def get_two_online_accounts(self):
-            ac1 = self.get_online_configuring_account()
-            ac2 = self.get_online_configuring_account()
-            wait_successful_IMAP_SMTP_connection(ac1)
-            wait_configuration_progress(ac1, 1000)
-            wait_successful_IMAP_SMTP_connection(ac2)
-            wait_configuration_progress(ac2, 1000)
-            return ac1, ac2
-
-        def clone_online_account(self, account):
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
-            ac.start_threads()
-            return ac
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
-    return am
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-    return Printer()
 
 
 def wait_configuration_progress(account, min_target, max_target=1001):
     min_target = min(min_target, max_target)
     while 1:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        if data1 >= min_target and data1 <= max_target:
+        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if event.data1 >= min_target and event.data1 <= max_target:
             print("** CONFIG PROGRESS {}".format(min_target), account)
             break
 
 
 def wait_securejoin_inviter_progress(account, target):
     while 1:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-        if data2 >= target:
+        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if event.data2 >= target:
             print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
             break
-
-
-def wait_successful_IMAP_SMTP_connection(account):
-    imap_ok = smtp_ok = False
-    while not imap_ok or not smtp_ok:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
-        if evt_name == "DC_EVENT_IMAP_CONNECTED":
-            imap_ok = True
-            print("** IMAP OK", account)
-        if evt_name == "DC_EVENT_SMTP_CONNECTED":
-            smtp_ok = True
-            print("** SMTP OK", account)
-    print("** IMAP and SMTP logins successful", account)
-
-
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev[1] == chat_id
-    if msg_id is not None:
-        assert ev[2] == msg_id
-    return ev[2]

python/tests/data/key

@@ -0,0 +1 @@
+../../../test-data/key

python/tests/test_increation.py

@@ -1,10 +1,57 @@
 from __future__ import print_function
+
+import os.path
+import shutil
+
+import pytest
 from filecmp import cmp
+
+from conftest import wait_configuration_progress
 from deltachat import const
-from conftest import wait_configuration_progress, wait_msgs_changed
+
+
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev.data1 == chat_id
+    if msg_id is not None:
+        assert ev.data2 == msg_id
+    return ev.data2
 
 
 class TestOnlineInCreation:
+    def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+
+        lp.sec("Creating in-creation file outside of blobdir")
+        assert tmpdir.strpath != ac1.get_blobdir()
+        src = tmpdir.join('file.txt').ensure(file=1)
+        with pytest.raises(Exception):
+            chat.prepare_message_file(src.strpath)
+
+    def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+
+        lp.sec("Creating file outside of blobdir")
+        assert tmpdir.strpath != ac1.get_blobdir()
+        src = tmpdir.join('file.txt')
+        src.write("hello there\n")
+        chat.send_file(src.strpath)
+
+        blob_src = os.path.join(ac1.get_blobdir(), 'file.txt')
+        assert os.path.exists(blob_src), "file.txt not copied to blobdir"
+
     def test_forward_increation(self, acfactory, data, lp):
         ac1 = acfactory.get_online_configuring_account()
         ac2 = acfactory.get_online_configuring_account()
@@ -17,7 +64,10 @@ class TestOnlineInCreation:
         wait_msgs_changed(ac1, 0, 0)  # why no chat id?
 
         lp.sec("create a message with a file in creation")
-        path = data.get_path("d.png")
+        orig = data.get_path("d.png")
+        path = os.path.join(ac1.get_blobdir(), 'd.png')
+        with open(path, "x") as fp:
+            fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -38,6 +88,7 @@ class TestOnlineInCreation:
 
         lp.sec("finish creating the file and send it")
         assert prepared_original.is_out_preparing()
+        shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -48,22 +99,22 @@ class TestOnlineInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat.id
-        assert ev[2] == prepared_original.id
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat2.id
-        assert ev[2] == forwarded_id
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat.id
+        assert ev.data2 == prepared_original.id
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat2.id
+        assert ev.data2 == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1[1] >= const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1[2])
-        assert cmp(received_original.filename, path, False)
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1.data2)
+        assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2[1] >= const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2[1] != ev1[1]
-        received_copy = ac2.get_message_by_id(ev2[2])
-        assert cmp(received_copy.filename, path, False)
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 != ev1.data1
+        received_copy = ac2.get_message_by_id(ev2.data2)
+        assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -1,8 +1,9 @@
 from __future__ import print_function
 from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
+from deltachat import register_global_plugin
+from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
-from deltachat.account import EventLogger
 
 
 def test_empty_context():
@@ -17,29 +18,31 @@ def test_callback_None2int():
     clear_context_callback(ctx)
 
 
-def test_dc_close_events(tmpdir):
-    ctx = ffi.gc(
-        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    evlog = EventLogger(ctx)
-    evlog.set_timeout(5)
-    set_context_callback(
-        ctx,
-        lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
-    )
-    p = tmpdir.join("hello.db")
-    lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
-    capi.lib.dc_close(ctx)
+def test_dc_close_events(tmpdir, acfactory):
+    ac1 = acfactory.get_unconfigured_account()
+
+    # register after_shutdown function
+    shutdowns = []
+
+    class ShutdownPlugin:
+        @global_hookimpl
+        def dc_account_after_shutdown(self, account):
+            assert account._dc_context is None
+            shutdowns.append(account)
+    register_global_plugin(ShutdownPlugin())
+    assert hasattr(ac1, "_dc_context")
+    ac1.shutdown()
+    assert shutdowns == [ac1]
 
     def find(info_string):
+        evlog = ac1._evtracker
         while 1:
             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
-            data2 = ev[2]
+            data2 = ev.data2
             if info_string in data2:
                 return
             else:
-                print("skipping event", *ev)
+                print("skipping event", ev)
 
     find("disconnecting inbox-thread")
     find("disconnecting sentbox-thread")
@@ -89,10 +92,10 @@ def test_markseen_invalid_message_ids(acfactory):
     contact1 = ac1.create_contact(email="some1@example.com", name="some1")
     chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
-    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
 def test_get_special_message_id_returns_empty_message(acfactory):
@@ -102,19 +105,12 @@ def test_get_special_message_id_returns_empty_message(acfactory):
         assert msg.id == 0
 
 
-def test_provider_info():
-    provider = lib.dc_provider_new_from_email(cutil.as_dc_charpointer("ex@example.com"))
-    assert cutil.from_dc_charpointer(
-        lib.dc_provider_get_overview_page(provider)
-    ) == "https://providers.delta.chat/example.com"
-    assert cutil.from_dc_charpointer(lib.dc_provider_get_name(provider)) == "Example"
-    assert cutil.from_dc_charpointer(lib.dc_provider_get_markdown(provider)) == "\n..."
-    assert cutil.from_dc_charpointer(lib.dc_provider_get_status_date(provider)) == "2018-09"
-    assert lib.dc_provider_get_status(provider) == const.DC_PROVIDER_STATUS_PREPARATION
-
-
 def test_provider_info_none():
-    assert lib.dc_provider_new_from_email(cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
 
 
 def test_get_info_closed():

python/tests/test_provider_info.py

@@ -1,27 +0,0 @@
-import pytest
-
-from deltachat import const
-from deltachat import provider
-
-
-def test_provider_info_from_email():
-    example = provider.Provider.from_email("email@example.com")
-    assert example.overview_page == "https://providers.delta.chat/example.com"
-    assert example.name == "Example"
-    assert example.markdown == "\n..."
-    assert example.status_date == "2018-09"
-    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
-
-
-def test_provider_info_from_domain():
-    example = provider.Provider("example.com")
-    assert example.overview_page == "https://providers.delta.chat/example.com"
-    assert example.name == "Example"
-    assert example.markdown == "\n..."
-    assert example.status_date == "2018-09"
-    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
-
-
-def test_provider_info_none():
-    with pytest.raises(provider.ProviderNotFoundError):
-        provider.Provider.from_email("email@unexistent.no")

python/tox.ini

@@ -7,17 +7,14 @@ envlist =
 
 [testenv]
 commands = 
-    # (some qr tests are pretty heavy in terms of send/received
-    # messages and async-imap's likely has concurrency problems, 
-    # eg https://github.com/async-email/async-imap/issues/4 )
-    pytest -n6 --reruns 3 --reruns-delay 5 -v -rsXx -k "not qr" {posargs:tests} 
-    pytest -n6 --reruns 5 --reruns-delay 5 -v -rsXx -k "qr" {posargs:tests} 
-    # python tests/package_wheels.py {toxworkdir}/wheelhouse
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
+    python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
+    DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -36,7 +33,7 @@ commands =
 
 [testenv:lint]
 skipsdist = True
-usedevelop = True
+skip_install = True
 deps =
     flake8
     # pygments required by rst-lint
@@ -44,13 +41,13 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/
+    flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx==2.2.0
+    sphinx
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
@@ -69,11 +66,10 @@ commands =
 
 
 [pytest]
-addopts = -v -rs 
-python_files = tests/test_*.py 
+addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 60 
+timeout = 90
 timeout_method = thread
 
 [flake8]

rust-toolchain

@@ -1 +1 @@
-nightly-2019-11-06
+nightly-2020-03-12

scripts/proxy.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# Examples:
+#
+# Original server that doesn't use SSL:
+# ./proxy.py 8080 imap.nauta.cu 143
+# ./proxy.py 8081 smtp.nauta.cu 25
+#
+# Original server that uses SSL:
+# ./proxy.py 8080 testrun.org 993 --ssl
+# ./proxy.py 8081 testrun.org 465 --ssl
+
+from datetime import datetime
+import argparse
+import selectors
+import ssl
+import socket
+import socketserver
+
+
+class Proxy(socketserver.ThreadingTCPServer):
+    allow_reuse_address = True
+
+    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
+        self.real_host = real_host
+        self.real_port = real_port
+        self.use_ssl = use_ssl
+        super().__init__((proxy_host, proxy_port), RequestHandler)
+
+
+class RequestHandler(socketserver.BaseRequestHandler):
+
+    def handle(self):
+        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
+
+        total = 0
+        real_server = (self.server.real_host, self.server.real_port)
+        with socket.create_connection(real_server) as sock:
+            if self.server.use_ssl:
+                context = ssl.create_default_context()
+                sock = context.wrap_socket(
+                    sock, server_hostname=real_server[0])
+
+            forward = {self.request: sock, sock: self.request}
+
+            sel = selectors.DefaultSelector()
+            sel.register(self.request, selectors.EVENT_READ,
+                         self.client_address)
+            sel.register(sock, selectors.EVENT_READ, real_server)
+
+            active = True
+            while active:
+                events = sel.select()
+                for key, mask in events:
+                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
+                    data = key.fileobj.recv(1024)
+                    received = len(data)
+                    total += received
+                    print(data)
+                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
+                    if data:
+                        forward[key.fileobj].sendall(data)
+                    else:
+                        print('\nCLOSING CONNECTION.\n\n')
+                        forward[key.fileobj].close()
+                        key.fileobj.close()
+                        active = False
+
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description='Simple Python Proxy')
+    p.add_argument(
+        "proxy_port", help="the port where the proxy will listen", type=int)
+    p.add_argument('host', help="the real host")
+    p.add_argument('port', help="the port of the real host", type=int)
+    p.add_argument("--ssl", help="use ssl to connect to the real host",
+                   action="store_true")
+    args = p.parse_args()
+
+    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
+        proxy.serve_forever()

set_core_version.py

@@ -33,7 +33,9 @@ def replace_toml_version(relpath, newversion):
 if __name__ == "__main__":
 
     if len(sys.argv) < 2:
-        raise SystemExit("need argument: new version, example 1.0.0-beta.27")
+        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
+            print("{}: {}".format(x, read_toml_version(x)))
+        raise SystemExit("need argument: new version, example: 1.25.0")
     newversion = sys.argv[1]
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
@@ -43,7 +45,7 @@ if __name__ == "__main__":
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
     for line in open("CHANGELOG.md"):
-        ## 1.0.0-beta5
+        ## 1.25.0
         if line.startswith("## "):
             if line[2:].strip().startswith(newversion):
                 break
@@ -53,7 +55,9 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
-    subprocess.call(["cargo", "update", "-p", "deltachat"])
+    subprocess.call(["cargo", "check"])
+    subprocess.call(["git", "add", "-u"])
+    # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
     print("after commit make sure to: ")
     print("")

spec.md

@@ -1,6 +1,6 @@
 # Chat-over-Email specification
 
-Version 0.19.0
+Version 0.30.0
 
 This document describes how emails can be used
 to implement typical messenger functions
@@ -17,6 +17,9 @@ while staying compatible to existing MUAs.
     - [Change group name](#change-group-name)
     - [Set group image](#set-group-image)
 - [Set profile image](#set-profile-image)
+- [Locations](#locations)
+    - [User locations](#user-locations)
+    - [Points of interest](#points-of-interest)
 - [Miscellaneous](#miscellaneous)
 
 
@@ -29,8 +32,7 @@ Messages SHOULD be encrypted by the
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
 by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
 If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string
-`Chat: Encrypted message` where the part after the colon MAY be localized.
+the subject of encrypted messages SHOULD be replaced by the string `...`.
 
 
 # Outgoing messages
@@ -113,6 +115,7 @@ but MUAs typically expose the sender in the UI.
 Groups are chats with usually more than one recipient,
 each defined by an email-address.
 The sender plus the recipients are the group members.
+All group members form the member list.
 
 To allow different groups with the same members,
 groups are identified by a group-id.
@@ -135,8 +138,7 @@ The group-name MUST be written to `Chat-Group-Name` header
 to join a group chat on a second device any time).
 
 The `Subject` header of outgoing group messages
-SHOULD start with the characters `Chat:`
-followed by the group-name and a colon followed by an excerpt of the message.
+SHOULD be set to the group-name.
 
 To identify the group-id on replies from normal MUAs,
 the group-id MUST also be added to the message-id of outgoing messages.
@@ -153,8 +155,8 @@ The message-id MUST have the format `Gr.<group-id>.<unique data>`.
     Hello group - this group contains three members
 
 Messengers adding the member list in the form `Name <email-address>`
-MUST take care only to spread the names authorized by the contacts themselves.
-Otherwise, names as _Daddy_ or _Honey_ may be spread
+MUST take care only to distribute the names authorized by the contacts themselves.
+Otherwise, names as _Daddy_ or _Honey_ may be distributed
 (this issue is also true for normal MUAs, however,
 for more contact- and chat-centralized apps
 such situations happen more frequently).
@@ -177,12 +179,22 @@ to a normal single-user chat with the email-address given in `From`.
 
 ## Add and remove members
 
-Messenger clients MUST construct the member list
-from the `From`/`To` headers only on the first group message
-or if they see a `Chat-Group-Member-Added`
-or `Chat-Group-Member-Removed` action header.
-Both headers MUST have the email-address
-of the added or removed member as the value.
+Messenger clients MUST init the member list
+from the `From`/`To` headers on the first group message.
+
+When a member is added later,
+a `Chat-Group-Member-Added` action header must be sent
+with the value set to the email-address of the added member.
+When receiving a `Chat-Group-Member-Added` header, however,
+_all missing_ members  the `From`/`To` headers has to be added.
+This is to mitigate problems when receiving messages
+in different orders, esp. on creating new groups.
+
+To remove a member, a `Chat-Group-Member-Removed` header must be sent
+with the value set to the email-address of the member to remove.
+When receiving a `Chat-Group-Member-Removed` header,
+only exaxtly the given member has to be removed from the member list.
+
 Messenger clients MUST NOT construct the member list
 on other group messages
 (this is to avoid accidentally altered To-lists in normal MUAs;
@@ -248,11 +260,11 @@ and the message SHOULD appear as a message or action from the sender.
 A group MAY have a group-image.
 To change or set the group-image,
 the messenger MUST attach an image file to a message
-and MUST add the header `Chat-Group-Image`
+and MUST add the header `Chat-Group-Avatar`
 with the value set to the image name.
 
 To remove the group-image,
-the messenger MUST add the header `Chat-Group-Image: 0`.
+the messenger MUST add the header `Chat-Group-Avatar: 0`.
 
 The messenger SHOULD send an explicit mail for each group image change.
 The body of the message SHOULD contain
@@ -265,7 +277,7 @@ and the message SHOULD appear as a message or action from the sender.
     Chat-Version: 1.0
     Chat-Group-ID: 12345uvwxyZ
     Chat-Group-Name: Our Group
-    Chat-Group-Image: image.jpg
+    Chat-Group-Avatar: image.jpg
     Message-ID: Gr.12345uvwxyZ.0005@domain
     Subject: Chat: Our Group: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
@@ -283,25 +295,25 @@ and the message SHOULD appear as a message or action from the sender.
 
 The image format SHOULD be image/jpeg or image/png.
 To save data, it is RECOMMENDED
-to add a `Chat-Group-Image` only on image changes.
+to add a `Chat-Group-Avatar` only on image changes.
 
 
 # Set profile image
 
-A user MAY have a profile-image that MAY be spread to his contacts.
+A user MAY have a profile-image that MAY be distributed to their contacts.
 To change or set the profile-image,
 the messenger MUST attach an image file to a message
-and MUST add the header `Chat-Profile-Image`
+and MUST add the header `Chat-User-Avatar`
 with the value set to the image name.
 
 To remove the profile-image,
-the messenger MUST add the header `Chat-Profile-Image: 0`.
+the messenger MUST add the header `Chat-User-Avatar: 0`.
 
-To spread the image,
+To distribute the image,
 the messenger MAY send the profile image
 together with the next mail to a given contact
 (to do this only once,
-the messenger has to keep a `profile_image_update_state` somewhere).
+the messenger has to keep a `user_avatar_update_state` somewhere).
 Alternatively, the messenger MAY send an explicit mail
 for each profile-image change to all contacts using a compatible messenger.
 The messenger SHOULD NOT send an explicit mail to normal MUAs.
@@ -309,7 +321,7 @@ The messenger SHOULD NOT send an explicit mail to normal MUAs.
     From: sender@domain
     To: rcpt@domain
     Chat-Version: 1.0
-    Chat-Profile-Image: photo.jpg
+    Chat-User-Avatar: photo.jpg
     Subject: Chat: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
 
@@ -325,13 +337,71 @@ The messenger SHOULD NOT send an explicit mail to normal MUAs.
     --==break==--
 
 The image format SHOULD be image/jpeg or image/png.
-Note that `Chat-Profile-Image` may appear together with all other headers,
-eg. there may be a `Chat-Profile-Image` and a `Chat-Group-Image` header
+Note that `Chat-User-Avatar` may appear together with all other headers,
+eg. there may be a `Chat-User-Avatar` and a `Chat-Group-Avatar` header
 in the same message.
-To save data, it is RECOMMENDED to add a `Chat-Profile-Image` header
+To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
 only on image changes.
 
 
+# Locations
+
+Locations can be attachted to messages using
+[standard kml-files](https://www.opengeospatial.org/standards/kml/)
+with well-known names.
+
+
+## User locations
+
+To send the location of the sender,
+the app can attach a file with the name `location.kml`.
+The file can contain one or more locations.
+Apps that support location streaming will typically collect some location events
+and send them together in one file.
+As each location has an independent timestamp,
+the apps can show the location as a track.
+
+Note that the `addr` attribute inside the  `location.kml` file
+MUST match the users email-address.
+Otherwise, the file is discarded silently;
+this is to protect against getting wrong locations,
+eg. forwarded from a normal MUA.
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <kml xmlns="http://www.opengis.net/kml/2.2">
+      <Document addr="ndh@deltachat.de">
+        <Placemark>
+          <Timestamp><when>2020-01-11T20:40:19Z</when></Timestamp>
+          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
+        </Placemark>
+        <Placemark>
+          <Timestamp><when>2020-01-11T20:40:25Z</when></Timestamp>
+          <Point><coordinates accuracy="5.4">7.654,3.21</coordinates></Point>
+        </Placemark>
+      </Document>
+    </kml>
+
+
+## Points of interest
+
+To send an "Point of interest", a POI,
+use a normal message and attach a file with the name  `message.kml`.
+In contrast to user locations, this file should contain only one location
+and an address-attribute is not needed -
+as the location belongs to the message content,
+it is fine if the location is detected on forwarding etc.
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <kml xmlns="http://www.opengis.net/kml/2.2">
+      <Document>
+        <Placemark>
+          <Timestamp><when>2020-01-01T20:40:19Z</when></Timestamp>
+          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
+        </Placemark>
+      </Document>
+    </kml>
+
+
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -368,4 +438,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2019 Delta Chat contributors.
+Copyright © 2017-2020 Delta Chat contributors.

src/aheader.rs

@@ -6,10 +6,10 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
-use crate::key::*;
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::{DcKey, SignedPublicKey};
 
 /// Possible values for encryption preference
 #[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive)]
@@ -52,13 +52,17 @@ impl str::FromStr for EncryptPreference {
 #[derive(Debug)]
 pub struct Aheader {
     pub addr: String,
-    pub public_key: Key,
+    pub public_key: SignedPublicKey,
     pub prefer_encrypt: EncryptPreference,
 }
 
 impl Aheader {
     /// Creates new autocrypt header
-    pub fn new(addr: String, public_key: Key, prefer_encrypt: EncryptPreference) -> Self {
+    pub fn new(
+        addr: String,
+        public_key: SignedPublicKey,
+        prefer_encrypt: EncryptPreference,
+    ) -> Self {
         Aheader {
             addr,
             public_key,
@@ -71,12 +75,9 @@ impl Aheader {
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
     ) -> Option<Self> {
-        use mailparse::MailHeaderMap;
-
-        if let Ok(Some(value)) = headers.get_first_value("Autocrypt") {
+        if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
             match Self::from_str(&value) {
                 Ok(header) => {
-                    info!(context, "comparing {} - {}", header.addr, wanted_from);
                     if addr_cmp(&header.addr, wanted_from) {
                         return Some(header);
                     }
@@ -96,16 +97,25 @@ impl Aheader {
 
 impl fmt::Display for Aheader {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        // TODO replace 78 with enum /rtn
-        // adds a whitespace every 78 characters, this allows libEtPan to
-        // wrap the lines according to RFC 5322
+        write!(fmt, "addr={};", self.addr)?;
+        if self.prefer_encrypt == EncryptPreference::Mutual {
+            write!(fmt, " prefer-encrypt=mutual;")?;
+        }
+
+        // adds a whitespace every 78 characters, this allows
+        // email crate to wrap the lines according to RFC 5322
         // (which may insert a linebreak before every whitespace)
-        let keydata = self.public_key.to_base64(78);
-        write!(
-            fmt,
-            "addr={}; prefer-encrypt={}; keydata={}",
-            self.addr, self.prefer_encrypt, keydata
-        )
+        let keydata = self.public_key.to_base64().chars().enumerate().fold(
+            String::new(),
+            |mut res, (i, c)| {
+                if i % 78 == 78 - "keydata=".len() {
+                    res.push(' ')
+                }
+                res.push(c);
+                res
+            },
+        );
+        write!(fmt, " keydata={}", keydata)
     }
 }
 
@@ -117,14 +127,10 @@ impl str::FromStr for Aheader {
             .split(';')
             .filter_map(|a| {
                 let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
-                if attribute.len() < 2 {
-                    return None;
+                match &attribute[..] {
+                    [key, value] => Some((key.trim().to_string(), value.trim().to_string())),
+                    _ => None,
                 }
-
-                Some((
-                    attribute[0].trim().to_string(),
-                    attribute[1].trim().to_string(),
-                ))
             })
             .collect();
 
@@ -134,30 +140,16 @@ impl str::FromStr for Aheader {
                 return Err(());
             }
         };
-
-        let public_key = match attributes
+        let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .and_then(|raw| Key::from_base64(&raw, KeyType::Public))
-        {
-            Some(key) => {
-                if key.verify() {
-                    key
-                } else {
-                    return Err(());
-                }
-            }
-            None => {
-                return Err(());
-            }
-        };
+            .ok_or(())
+            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
+            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
 
-        let prefer_encrypt = match attributes
+        let prefer_encrypt = attributes
             .remove("prefer-encrypt")
             .and_then(|raw| raw.parse().ok())
-        {
-            Some(pref) => pref,
-            None => EncryptPreference::NoPreference,
-        };
+            .unwrap_or_default();
 
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
@@ -177,15 +169,13 @@ impl str::FromStr for Aheader {
 mod tests {
     use super::*;
 
-    fn rawkey() -> String {
-        "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=".into()
-    }
+    const RAWKEY: &str = "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
 
     #[test]
     fn test_from_str() {
         let h: Aheader = format!(
             "addr=me@mail.com; prefer-encrypt=mutual; keydata={}",
-            rawkey()
+            RAWKEY
         )
         .parse()
         .expect("failed to parse");
@@ -194,9 +184,22 @@ mod tests {
         assert_eq!(h.prefer_encrypt, EncryptPreference::Mutual);
     }
 
+    // EncryptPreference::Reset is an internal value, parser should never return it
+    #[test]
+    fn test_from_str_reset() {
+        let raw = format!(
+            "addr=reset@example.com; prefer-encrypt=reset; keydata={}",
+            RAWKEY
+        );
+        let h: Aheader = raw.parse().expect("failed to parse");
+
+        assert_eq!(h.addr, "reset@example.com");
+        assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
+    }
+
     #[test]
     fn test_from_str_non_critical() {
-        let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", rawkey());
+        let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", RAWKEY);
         let h: Aheader = raw.parse().expect("failed to parse");
 
         assert_eq!(h.addr, "me@mail.com");
@@ -207,33 +210,57 @@ mod tests {
     fn test_from_str_superflous_critical() {
         let raw = format!(
             "addr=me@mail.com; _foo=one; _bar=two; other=me; keydata={}",
-            rawkey()
+            RAWKEY
         );
         assert!(raw.parse::<Aheader>().is_err());
     }
 
     #[test]
     fn test_good_headers() {
-        let fixed_header = "addr=a@b.example.org; prefer-encrypt=mutual; keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g 4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8 ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu88 80iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTll HOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+ws CJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiML AAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiF Nyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAg dLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72 rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v 81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgC u3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOt kb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKC LhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4s WVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWv BuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiML AAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGr wdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktE k6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0 j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7x egRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
+        let fixed_header = concat!(
+            "addr=a@b.example.org; prefer-encrypt=mutual; ",
+            "keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJg",
+            " WL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6",
+            " CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKK",
+            " bhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1Kv",
+            " VL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbG",
+            " UuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaK",
+            " rc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087",
+            " LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VN",
+            " HtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Ddd",
+            " fxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCv",
+            " SJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vau",
+            " f1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+",
+            " G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjm",
+            " kRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09",
+            " /JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHR",
+            " TR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaK",
+            " rc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivX",
+            " urm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9Mtrm",
+            " ZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb",
+            " +F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNg",
+            " wm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc="
+        );
 
         let ah = Aheader::from_str(fixed_header).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
+        assert_eq!(format!("{}", ah), fixed_header);
 
         let rendered = ah.to_string();
         assert_eq!(rendered, fixed_header);
 
-        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", rawkey())).expect("failed to parse");
+        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY)).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
 
         Aheader::from_str(&format!(
             "addr=a@b.example.org; prefer-encrypt=ignoreUnknownValues; keydata={}",
-            rawkey()
+            RAWKEY
         ))
         .expect("failed to parse");
 
-        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", rawkey()))
+        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))
             .expect("failed to parse");
     }
 
@@ -245,4 +272,30 @@ mod tests {
         assert!(Aheader::from_str(" ;;").is_err());
         assert!(Aheader::from_str("addr=a@t.de; unknwon=1; keydata=jau").is_err());
     }
+
+    #[test]
+    fn test_display_aheader() {
+        assert!(format!(
+            "{}",
+            Aheader::new(
+                "test@example.com".to_string(),
+                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                EncryptPreference::Mutual
+            )
+        )
+        .contains("prefer-encrypt=mutual;"));
+
+        // According to Autocrypt Level 1 specification,
+        // only "prefer-encrypt=mutual;" can be used.
+        // If the setting is nopreference, the whole attribute is omitted.
+        assert!(!format!(
+            "{}",
+            Aheader::new(
+                "test@example.com".to_string(),
+                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                EncryptPreference::NoPreference
+            )
+        )
+        .contains("prefer-encrypt"));
+    }
 }

src/blob.rs

@@ -6,6 +6,10 @@ use std::fs;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
+use image::GenericImageView;
+use thiserror::Error;
+
+use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
 use crate::events::Event;
 
@@ -52,7 +56,6 @@ impl<'a> BlobObject<'a> {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
                 cause: err,
-                backtrace: failure::Backtrace::new(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -80,7 +83,6 @@ impl<'a> BlobObject<'a> {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
-                            backtrace: failure::Backtrace::new(),
                         });
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -93,7 +95,6 @@ impl<'a> BlobObject<'a> {
             blobdir: dir.to_path_buf(),
             blobname: name,
             cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
-            backtrace: failure::Backtrace::new(),
         })
     }
 
@@ -118,7 +119,6 @@ impl<'a> BlobObject<'a> {
             blobname: String::from(""),
             src: src.as_ref().to_path_buf(),
             cause: err,
-            backtrace: failure::Backtrace::new(),
         })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
@@ -134,7 +134,6 @@ impl<'a> BlobObject<'a> {
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
-                backtrace: failure::Backtrace::new(),
             }
         })?;
         let blob = BlobObject {
@@ -159,7 +158,7 @@ impl<'a> BlobObject<'a> {
     /// This merely delegates to the [BlobObject::create_and_copy] and
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
-    pub fn create_from_path(
+    pub fn new_from_path(
         context: &Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject, BlobError> {
@@ -194,17 +193,14 @@ impl<'a> BlobObject<'a> {
             .map_err(|_| BlobError::WrongBlobdir {
                 blobdir: context.get_blobdir().to_path_buf(),
                 src: path.as_ref().to_path_buf(),
-                backtrace: failure::Backtrace::new(),
             })?;
         if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
                 blobname: path.as_ref().to_path_buf(),
-                backtrace: failure::Backtrace::new(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
             blobname: path.as_ref().to_path_buf(),
-            backtrace: failure::Backtrace::new(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -232,7 +228,6 @@ impl<'a> BlobObject<'a> {
         if !BlobObject::is_acceptible_blob_name(&name) {
             return Err(BlobError::WrongName {
                 blobname: PathBuf::from(name),
-                backtrace: failure::Backtrace::new(),
             });
         }
         Ok(BlobObject {
@@ -318,13 +313,12 @@ impl<'a> BlobObject<'a> {
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
         let mut iter = clean.splitn(2, '.');
-        let mut stem = iter.next().unwrap_or_default().to_string();
-        let mut ext = iter.next().unwrap_or_default().to_string();
-        stem.truncate(64);
-        ext.truncate(32);
-        match ext.len() {
-            0 => (stem, "".to_string()),
-            _ => (stem, format!(".{}", ext).to_lowercase()),
+        let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
+        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
+        if ext.is_empty() {
+            (stem, "".to_string())
+        } else {
+            (stem, format!(".{}", ext).to_lowercase())
         }
     }
 
@@ -349,6 +343,29 @@ impl<'a> BlobObject<'a> {
         }
         true
     }
+
+    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
+            return Ok(());
+        }
+
+        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        Ok(())
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -358,88 +375,41 @@ impl<'a> fmt::Display for BlobObject<'a> {
 }
 
 /// Errors for the [BlobObject].
-#[derive(Fail, Debug)]
+#[derive(Debug, Error)]
 pub enum BlobError {
+    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
     CreateFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
     WriteFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
         blobdir: PathBuf,
         blobname: String,
         src: PathBuf,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
-    WrongBlobdir {
+    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
+    RecodeFailure {
         blobdir: PathBuf,
-        src: PathBuf,
-        backtrace: failure::Backtrace,
+        blobname: String,
+        #[source]
+        cause: image::ImageError,
     },
-    WrongName {
-        blobname: PathBuf,
-        backtrace: failure::Backtrace,
-    },
-}
-
-// Implementing Display is done by hand because the failure
-// #[fail(display = "...")] syntax does not allow using
-// `blobdir.display()`.
-impl fmt::Display for BlobError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        // Match on the data rather than kind, they are equivalent for
-        // identifying purposes but contain the actual data we need.
-        match &self {
-            BlobError::CreateFailure {
-                blobdir, blobname, ..
-            } => write!(
-                f,
-                "Failed to create blob {} in {}",
-                blobname,
-                blobdir.display()
-            ),
-            BlobError::WriteFailure {
-                blobdir, blobname, ..
-            } => write!(
-                f,
-                "Failed to write data to blob {} in {}",
-                blobname,
-                blobdir.display()
-            ),
-            BlobError::CopyFailure {
-                blobdir,
-                blobname,
-                src,
-                ..
-            } => write!(
-                f,
-                "Failed to copy data from {} to blob {} in {}",
-                src.display(),
-                blobname,
-                blobdir.display(),
-            ),
-            BlobError::WrongBlobdir { blobdir, src, .. } => write!(
-                f,
-                "File path {} is not in blobdir {}",
-                src.display(),
-                blobdir.display(),
-            ),
-            BlobError::WrongName { blobname, .. } => {
-                write!(f, "Blob has a bad name: {}", blobname.display(),)
-            }
-        }
-    }
+    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
+    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
+    #[error("Blob has a badname {}", .blobname.display())]
+    WrongName { blobname: PathBuf },
 }
 
 #[cfg(test)]
@@ -483,23 +453,23 @@ mod tests {
     #[test]
     fn test_suffix() {
         let t = dummy_context();
-        let foo = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
-        assert_eq!(foo.suffix(), Some("txt"));
-        let bar = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
-        assert_eq!(bar.suffix(), None);
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.suffix(), Some("txt"));
+        let blob = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
+        assert_eq!(blob.suffix(), None);
     }
 
     #[test]
     fn test_create_dup() {
         let t = dummy_context();
         BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
-        let foo = t.ctx.get_blobdir().join("foo.txt");
-        assert!(foo.exists());
+        let foo_path = t.ctx.get_blobdir().join("foo.txt");
+        assert!(foo_path.exists());
         BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
         for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
             let fname = dirent.unwrap().file_name();
-            if fname == foo.file_name().unwrap() {
-                assert_eq!(fs::read(&foo).unwrap(), b"hello");
+            if fname == foo_path.file_name().unwrap() {
+                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 assert!(name.starts_with("foo"));
@@ -512,13 +482,13 @@ mod tests {
     fn test_double_ext_preserved() {
         let t = dummy_context();
         BlobObject::create(&t.ctx, "foo.tar.gz", b"hello").unwrap();
-        let foo = t.ctx.get_blobdir().join("foo.tar.gz");
-        assert!(foo.exists());
+        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
+        assert!(foo_path.exists());
         BlobObject::create(&t.ctx, "foo.tar.gz", b"world").unwrap();
         for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
             let fname = dirent.unwrap().file_name();
-            if fname == foo.file_name().unwrap() {
-                assert_eq!(fs::read(&foo).unwrap(), b"hello");
+            if fname == foo_path.file_name().unwrap() {
+                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 println!("{}", name);
@@ -559,14 +529,14 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
 
         let src_int = t.ctx.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").unwrap();
-        let blob = BlobObject::create_from_path(&t.ctx, &src_int).unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
@@ -576,7 +546,7 @@ mod tests {
         let t = dummy_context();
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -592,4 +562,11 @@ mod tests {
         assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
         assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
     }
+
+    #[test]
+    fn test_sanitise_name() {
+        let (_, ext) =
+            BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
+        assert_eq!(ext, ".txt");
+    }
 }

src/chatlist.rs

@@ -1,10 +1,11 @@
 //! # Chat list module
 
+use crate::chat;
 use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
-use crate::error::Result;
+use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
@@ -36,7 +37,7 @@ use crate::stock::StockMessage;
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(u32, MsgId)>,
+    ids: Vec<(ChatId, MsgId)>,
 }
 
 impl Chatlist {
@@ -60,7 +61,7 @@ impl Chatlist {
     ///   or "Not now".
     ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
-    ///   archived _any_ chat using dc_archive_chat(). The UI should show a link as
+    ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
     ///   list of all archived chats that can be created by this function hen using
     ///   the DC_GCL_ARCHIVED_ONLY flag.
@@ -71,8 +72,11 @@ impl Chatlist {
     /// The `listflags` is a combination of flags:
     /// - if the flag DC_GCL_ARCHIVED_ONLY is set, only archived chats are returned.
     ///   if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
-    ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived
+    ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
+    /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
+    ///   and hides the device-chat,
+    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -88,10 +92,16 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
+        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
+        // messages get deleted to avoid reloading the same chatlist.
+        if let Err(err) = delete_device_expired_messages(context) {
+            warn!(context, "Failed to hide expired messages: {}", err);
+        }
+
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
-            let chat_id: u32 = row.get(0)?;
+            let chat_id: ChatId = row.get(0)?;
             let msg_id: MsgId = row.get(1).unwrap_or_default();
             Ok((chat_id, msg_id))
         };
@@ -101,6 +111,14 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
+        let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+                .unwrap_or_default()
+                .0
+        } else {
+            ChatId::new(0)
+        };
+
         // select with left join and minimum:
         //
         // - the inner select must use `hidden` and _not_ `m.hidden`
@@ -127,18 +145,21 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?1))
                  WHERE c.id>9
                    AND c.blocked=0
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, query_contact_id as i32],
+                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             )?
         } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
+            // (this includes the archived device-chat; we could skip it,
+            // however, then the number of archived chats do not match, which might be even more irritating.
+            // and adapting the number requires larger refactorings and seems not to be worth the effort)
             context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
@@ -162,6 +183,12 @@ impl Chatlist {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
 
+            // allow searching over special names that may change at any time
+            // when the ui calls set_stock_translation()
+            if let Err(err) = update_special_chat_names(context) {
+                warn!(context, "cannot update special chat names: {:?}", err)
+            }
+
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
                 "SELECT c.id, m.id
@@ -172,18 +199,25 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
-                 WHERE c.id>9
+                                  AND (hidden=0 OR state=?1))
+                 WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
-                   AND c.name LIKE ?
+                   AND c.name LIKE ?3
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, str_like_cmd],
+                params![MessageState::OutDraft, skip_id, str_like_cmd],
                 process_row,
                 process_rows,
             )?
         } else {
             //  show normal chatlist
+            let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+                    .unwrap_or_default()
+                    .0
+            } else {
+                ChatId::new(0)
+            };
             let mut ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
@@ -193,19 +227,24 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
-                 WHERE c.id>9
+                                  AND (hidden=0 OR state=?1))
+                 WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
-                   AND c.archived=0
+                   AND NOT c.archived=?3
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft],
+                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             )?;
             if 0 == listflags & DC_GCL_NO_SPECIALS {
                 if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
+                    if 0 == listflags & DC_GCL_FOR_FORWARDING {
+                        ids.insert(
+                            0,
+                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
+                        );
+                    }
                 }
                 add_archived_link_item = true;
             }
@@ -214,9 +253,9 @@ impl Chatlist {
 
         if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
             if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
+                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
+            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -235,19 +274,21 @@ impl Chatlist {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> u32 {
-        if index >= self.ids.len() {
-            return 0;
+    pub fn get_chat_id(&self, index: usize) -> ChatId {
+        match self.ids.get(index) {
+            Some((chat_id, _msg_id)) => *chat_id,
+            None => ChatId::new(0),
         }
-        self.ids[index].0
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
-        ensure!(index < self.ids.len(), "Chatlist index out of range");
-        Ok(self.ids[index].1)
+        match self.ids.get(index) {
+            Some((_chat_id, msg_id)) => Ok(*msg_id),
+            None => bail!("Chatlist index out of range"),
+        }
     }
 
     /// Get a summary for a chatlist index.
@@ -269,27 +310,29 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-
         let mut ret = Lot::new();
-        if index >= self.ids.len() {
-            ret.text2 = Some("ErrBadChatlistIndex".to_string());
-            return ret;
-        }
+
+        let (chat_id, lastmsg_id) = match self.ids.get(index) {
+            Some(ids) => ids,
+            None => {
+                ret.text2 = Some("ErrBadChatlistIndex".to_string());
+                return ret;
+            }
+        };
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
+        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) {
             chat_loaded = chat;
             &chat_loaded
         } else {
             return ret;
         };
 
-        let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
@@ -301,7 +344,7 @@ impl Chatlist {
             None
         };
 
-        if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
+        if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
@@ -312,9 +355,13 @@ impl Chatlist {
 
         ret
     }
+
+    pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
+        self.ids.iter().position(|(chat_id, _)| chat_id == &id)
+    }
 }
 
-/// Get the number of archived chats
+/// Returns the number of archived chats
 pub fn dc_get_archived_cnt(context: &Context) -> u32 {
     context
         .sql
@@ -349,7 +396,6 @@ fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
 mod tests {
     use super::*;
 
-    use crate::chat;
     use crate::test_utils::*;
 
     #[test]
@@ -369,7 +415,7 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        set_draft(&t.ctx, chat_id2, Some(&mut msg));
+        chat_id2.set_draft(&t.ctx, Some(&mut msg));
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
@@ -380,8 +426,66 @@ mod tests {
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 0);
 
-        chat::archive(&t.ctx, chat_id1, true).ok();
+        chat_id1
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .ok();
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 1);
     }
+
+    #[test]
+    fn test_sort_self_talk_up_on_forward() {
+        let t = dummy_context();
+        t.ctx.update_device_chats().unwrap();
+        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert!(chats.len() == 3);
+        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .unwrap()
+            .is_self_talk());
+
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None).unwrap();
+        assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
+        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .unwrap()
+            .is_self_talk());
+    }
+
+    #[test]
+    fn test_search_special_chat_names() {
+        let t = dummy_context();
+        t.ctx.update_device_chats().unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        assert_eq!(chats.len(), 0);
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        assert_eq!(chats.len(), 0);
+
+        t.ctx
+            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+
+        t.ctx
+            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+    }
+
+    #[test]
+    fn test_get_summary_unwrap() {
+        let t = dummy_context();
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("foo:\nbar \r\n test".to_string()));
+        chat_id1.set_draft(&t.ctx, Some(&mut msg));
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let summary = chats.get_summary(&t.ctx, 0, None);
+        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
+    }
 }

src/config.rs

@@ -4,11 +4,16 @@ use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
+use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::events::Event;
 use crate::job::*;
+use crate::message::MsgId;
+use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
+use rusqlite::NO_PARAMS;
 
 /// The available configuration keys.
 #[derive(
@@ -60,6 +65,28 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
+    #[strum(props(default = "0"))]
+    KeyGenType,
+
+    /// Timer in seconds after which the message is deleted from the
+    /// server.
+    ///
+    /// Equals to 0 by default, which means the message is never
+    /// deleted.
+    ///
+    /// Value 1 is treated as "delete at once": messages are deleted
+    /// immediately, without moving to DeltaChat folder.
+    #[strum(props(default = "0"))]
+    DeleteServerAfter,
+
+    /// Timer in seconds after which the message is deleted from the
+    /// device.
+    ///
+    /// Equals to 0 by default, which means the message is never
+    /// deleted.
+    #[strum(props(default = "0"))]
+    DeleteDeviceAfter,
+
     SaveMimeHeaders,
     ConfiguredAddr,
     ConfiguredMailServer,
@@ -97,7 +124,7 @@ impl Context {
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
-            Config::SysMsgsizeMaxRecommended => Some(format!("{}", 24 * 1024 * 1024 / 4 * 3)),
+            Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
             _ => self.sql.get_raw_config(self, key),
         };
@@ -123,17 +150,50 @@ impl Context {
         self.get_config_int(key) != 0
     }
 
+    /// Gets configured "delete_server_after" value.
+    ///
+    /// `None` means never delete the message, `Some(0)` means delete
+    /// at once, `Some(x)` means delete after `x` seconds.
+    pub fn get_config_delete_server_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteServerAfter) {
+            0 => None,
+            1 => Some(0),
+            x => Some(x as i64),
+        }
+    }
+
+    /// Gets configured "delete_device_after" value.
+    ///
+    /// `None` means never delete the message, `Some(x)` means delete
+    /// after `x` seconds.
+    pub fn get_config_delete_device_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteDeviceAfter) {
+            0 => None,
+            x => Some(x as i64),
+        }
+    }
+
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
     pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
-            Config::Selfavatar if value.is_some() => {
-                let blob = BlobObject::create_from_path(&self, value.unwrap())?;
-                self.sql.set_raw_config(self, key, Some(blob.as_name()))
+            Config::Selfavatar => {
+                self.sql
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", NO_PARAMS)?;
+                self.sql
+                    .set_raw_config_bool(self, "attach_selfavatar", true)?;
+                match value {
+                    Some(value) => {
+                        let blob = BlobObject::new_from_path(&self, value)?;
+                        blob.recode_to_avatar_size(self)?;
+                        self.sql.set_raw_config(self, key, Some(blob.as_name()))
+                    }
+                    None => self.sql.set_raw_config(self, key, None),
+                }
             }
             Config::InboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_inbox_idle(self, true);
+                interrupt_inbox_idle(self);
                 ret
             }
             Config::SentboxWatch => {
@@ -156,6 +216,15 @@ impl Context {
 
                 self.sql.set_raw_config(self, key, val)
             }
+            Config::DeleteDeviceAfter => {
+                let ret = self.sql.set_raw_config(self, key, value);
+                // Force chatlist reload to delete old messages immediately.
+                self.call_cb(Event::MsgsChanged {
+                    msg_id: MsgId::new(0),
+                    chat_id: ChatId::new(0),
+                });
+                ret
+            }
             _ => self.sql.set_raw_config(self, key, value),
         }
     }
@@ -179,7 +248,11 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
+    use image::GenericImageView;
+    use std::fs::File;
+    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -199,30 +272,78 @@ mod tests {
     }
 
     #[test]
-    fn test_selfavatar() -> failure::Fallible<()> {
+    fn test_selfavatar_outside_blobdir() {
         let t = dummy_context();
         let avatar_src = t.dir.path().join("avatar.jpg");
-        std::fs::write(&avatar_src, b"avatar")?;
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
         let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
         assert!(!avatar_blob.exists());
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .unwrap();
         assert!(avatar_blob.exists());
-        assert_eq!(std::fs::read(&avatar_blob)?, b"avatar");
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
         let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-        Ok(())
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(avatar_blob).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
     }
 
     #[test]
-    fn test_selfavatar_in_blobdir() -> failure::Fallible<()> {
+    fn test_selfavatar_in_blobdir() {
         let t = dummy_context();
-        let avatar_src = t.ctx.get_blobdir().join("avatar.jpg");
-        std::fs::write(&avatar_src, b"avatar")?;
+        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .unwrap();
         let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
-        Ok(())
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
+    }
+
+    #[test]
+    fn test_selfavatar_copy_without_recode() {
+        let t = dummy_context();
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists());
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .unwrap();
+        assert!(avatar_blob.exists());
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
 }

src/configure/auto_mozilla.rs

@@ -1,9 +1,6 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
 //! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-use failure::Fail;
-
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
@@ -11,33 +8,7 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "Invalid email address: {:?}", _0)]
-    InvalidEmailAddress(String),
-
-    #[fail(display = "XML error at position {}", position)]
-    InvalidXml {
-        position: usize,
-        #[cause]
-        error: quick_xml::Error,
-    },
-
-    #[fail(display = "Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[fail(display = "Failed to get URL {}", _0)]
-    ReadUrlError(#[cause] super::read_url::Error),
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<super::read_url::Error> for Error {
-    fn from(err: super::read_url::Error) -> Error {
-        Error::ReadUrlError(err)
-    }
-}
+use super::Error;
 
 #[derive(Debug)]
 struct MozAutoconfigure<'a> {
@@ -67,16 +38,16 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam> {
+fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     // Split address into local part and domain part.
-    let p = in_emailaddr
-        .find('@')
-        .ok_or(Error::InvalidEmailAddress(in_emailaddr.to_string()))?;
-    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
-    let in_emaildomain = &in_emaildomain[1..];
+    let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
+    let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
+        [domain, local] => (local, domain),
+        _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
+    };
 
     let mut moz_ac = MozAutoconfigure {
         in_emailaddr,
@@ -127,16 +98,17 @@ pub fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam> {
+) -> Result<LoginParam, Error> {
     let xml_raw = read_url(context, url)?;
 
-    parse_xml(&param_in.addr, &xml_raw).map_err(|err| {
+    let res = parse_xml(&param_in.addr, &xml_raw);
+    if let Err(err) = &res {
         warn!(
             context,
             "Failed to parse Thunderbird autoconfiguration XML: {}", err
         );
-        err.into()
-    })
+    }
+    res
 }
 
 fn moz_autoconfigure_text_cb<B: std::io::BufRead>(

src/configure/auto_outlook.rs

@@ -1,8 +1,5 @@
 //! Outlook's Autodiscover
 
-use failure::Fail;
-
-use quick_xml;
 use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
@@ -10,33 +7,7 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "XML error at position {}", position)]
-    InvalidXml {
-        position: usize,
-        #[cause]
-        error: quick_xml::Error,
-    },
-
-    #[fail(display = "Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[fail(display = "Failed to get URL {}", _0)]
-    ReadUrlError(#[cause] super::read_url::Error),
-
-    #[fail(display = "Number of redirection is exceeded")]
-    RedirectionError,
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<super::read_url::Error> for Error {
-    fn from(err: super::read_url::Error) -> Error {
-        Error::ReadUrlError(err)
-    }
-}
+use super::Error;
 
 struct OutlookAutodiscover {
     pub out: LoginParam,
@@ -54,7 +25,7 @@ enum ParsingResult {
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult> {
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
         out_imap_set: false,
@@ -145,7 +116,7 @@ pub fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
-) -> Result<LoginParam> {
+) -> Result<LoginParam, Error> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {

src/configure/mod.rs

@@ -6,16 +6,20 @@ mod read_url;
 
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
+use async_std::task;
+
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::e2ee;
-use crate::job::*;
-use crate::login_param::LoginParam;
+use crate::error::format_err;
+use crate::job::{self, job_add, job_kill_action};
+use crate::login_param::{CertificateChecks, LoginParam};
 use crate::oauth2::*;
 use crate::param::Params;
+use crate::{chat, e2ee, provider};
 
+use crate::message::Message;
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
 
@@ -29,34 +33,36 @@ macro_rules! progress {
     };
 }
 
-// connect
-pub fn configure(context: &Context) {
-    if context.has_ongoing() {
-        warn!(context, "There is already another ongoing process running.",);
-        return;
+impl Context {
+    /// Starts a configuration job.
+    pub fn configure(&self) {
+        if self.has_ongoing() {
+            warn!(self, "There is already another ongoing process running.",);
+            return;
+        }
+        job_kill_action(self, job::Action::ConfigureImap);
+        job_add(self, job::Action::ConfigureImap, 0, Params::new(), 0);
     }
-    job_kill_action(context, Action::ConfigureImap);
-    job_add(context, Action::ConfigureImap, 0, Params::new(), 0);
-}
 
-/// Check if the context is already configured.
-pub fn dc_is_configured(context: &Context) -> bool {
-    context.sql.get_raw_config_bool(context, "configured")
+    /// Checks if the context is already configured.
+    pub fn is_configured(&self) -> bool {
+        self.sql.get_raw_config_bool(self, "configured")
+    }
 }
 
 /*******************************************************************************
  * Configure JOB
  ******************************************************************************/
-#[allow(non_snake_case, unused_must_use)]
-pub fn JobConfigureImap(context: &Context) {
+#[allow(non_snake_case, unused_must_use, clippy::cognitive_complexity)]
+pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     if !context.sql.is_open() {
         error!(context, "Cannot configure, database not opened.",);
         progress!(context, 0);
-        return;
+        return job::Status::Finished(Err(format_err!("Database not opened")));
     }
     if !context.alloc_ongoing() {
         progress!(context, 0);
-        return;
+        return job::Status::Finished(Err(format_err!("Cannot allocated ongoing process")));
     }
     let mut success = false;
     let mut imap_connected_here = false;
@@ -92,9 +98,11 @@ pub fn JobConfigureImap(context: &Context) {
     let mut param_domain = "undefined.undefined".to_owned();
     let mut param_addr_urlencoded: String =
         "Internal Error: this value should never be used".to_owned();
-    let mut keep_flags = std::i32::MAX;
+    let mut keep_flags = 0;
+
+    const STEP_12_USE_AUTOCONFIG: u8 = 12;
+    const STEP_13_AFTER_AUTOCONFIG: u8 = 13;
 
-    const STEP_3_INDEX: u8 = 13;
     let mut step_counter: u8 = 0;
     while !context.shall_stop_ongoing() {
         step_counter += 1;
@@ -110,7 +118,7 @@ pub fn JobConfigureImap(context: &Context) {
             }
             // Step 1: Load the parameters and check email-address and password
             2 => {
-                if 0 != param.server_flags & 0x2 {
+                if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
                     // the used oauth2 addr may differ, check this.
                     // if dc_get_oauth2_addr() is not available in the oauth2 implementation,
                     // just use the given one.
@@ -145,6 +153,7 @@ pub fn JobConfigureImap(context: &Context) {
             // Step 2: Autoconfig
             4 => {
                 progress!(context, 200);
+
                 if param.mail_server.is_empty()
                             && param.mail_port == 0
                             /*&&param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then */
@@ -152,12 +161,18 @@ pub fn JobConfigureImap(context: &Context) {
                             && param.send_port == 0
                             && param.send_user.is_empty()
                             /*&&param.send_pw.is_empty() -- the password cannot be auto-configured and is no criterion for autoconfig or not */
-                            && param.server_flags & !0x2 == 0
+                            && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
                 {
-                    keep_flags = param.server_flags & 0x2;
+                    // no advanced parameters entered by the user: query provider-database or do Autoconfig
+                    keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
+                    if let Some(new_param) = get_offline_autoconfig(context, &param) {
+                        // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
+                        param_autoconfig = Some(new_param);
+                        step_counter = STEP_12_USE_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
+                    }
                 } else {
-                    // Autoconfig is not needed so skip it.
-                    step_counter = STEP_3_INDEX - 1;
+                    // advanced parameters entered by the user: skip Autoconfig
+                    step_counter = STEP_13_AFTER_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
                 }
                 true
             }
@@ -189,10 +204,7 @@ pub fn JobConfigureImap(context: &Context) {
             7 => {
                 progress!(context, 310);
                 if param_autoconfig.is_none() {
-                    let url = format!(
-                        "https://{}{}/autodiscover/autodiscover.xml",
-                        "", param_domain
-                    );
+                    let url = format!("https://{}/autodiscover/autodiscover.xml", param_domain);
                     param_autoconfig = outlk_autodiscover(context, &url, &param).ok();
                 }
                 true
@@ -242,8 +254,10 @@ pub fn JobConfigureImap(context: &Context) {
                 }
                 true
             }
-            /* C.  Do we have any result? */
-            12 => {
+            /* C.  Do we have any autoconfig result?
+               If you change the match-number here, also update STEP_12_COPY_AUTOCONFIG above
+            */
+            STEP_12_USE_AUTOCONFIG => {
                 progress!(context, 500);
                 if let Some(ref cfg) = param_autoconfig {
                     info!(context, "Got autoconfig: {}", &cfg);
@@ -256,15 +270,15 @@ pub fn JobConfigureImap(context: &Context) {
                     param.send_port = cfg.send_port;
                     param.send_user = cfg.send_user.clone();
                     param.server_flags = cfg.server_flags;
-                    /* although param_autoconfig's data are no longer needed from, it is important to keep the object as
-                    we may enter "deep guessing" if we could not read a configuration */
+                    /* although param_autoconfig's data are no longer needed from,
+                    it is used to later to prevent trying variations of port/server/logins */
                 }
                 param.server_flags |= keep_flags;
                 true
             }
             // Step 3: Fill missing fields with defaults
-            13 => {
-                // if you move this, don't forget to update STEP_3_INDEX, too
+            // If you change the match-number here, also update STEP_13_AFTER_AUTOCONFIG above
+            STEP_13_AFTER_AUTOCONFIG => {
                 if param.mail_server.is_empty() {
                     param.mail_server = format!("imap.{}", param_domain,)
                 }
@@ -358,7 +372,7 @@ pub fn JobConfigureImap(context: &Context) {
                 let create_mvbox = context.get_config_bool(Config::MvboxWatch)
                     || context.get_config_bool(Config::MvboxMove);
                 let imap = &context.inbox_thread.read().unwrap().imap;
-                if let Err(err) = imap.ensure_configured_folders(context, create_mvbox) {
+                if let Err(err) = imap.configure_folders(context, create_mvbox) {
                     warn!(context, "configuring folders failed: {:?}", err);
                     false
                 } else {
@@ -426,8 +440,77 @@ pub fn JobConfigureImap(context: &Context) {
         LoginParam::from_database(context, "configured_raw_").save_to_database(context, "");
     }
 
+    if let Some(provider) = provider::get_provider_info(&param.addr) {
+        if !provider.after_login_hint.is_empty() {
+            let mut msg = Message::new(Viewtype::Text);
+            msg.text = Some(provider.after_login_hint.to_string());
+            if chat::add_device_msg(context, Some("core-provider-info"), Some(&mut msg)).is_err() {
+                warn!(context, "cannot add after_login_hint as core-provider-info");
+            }
+        }
+    }
+
     context.free_ongoing();
     progress!(context, if success { 1000 } else { 0 });
+    job::Status::Finished(Ok(()))
+}
+
+#[allow(clippy::unnecessary_unwrap)]
+fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
+    info!(
+        context,
+        "checking internal provider-info for offline autoconfig"
+    );
+
+    if let Some(provider) = provider::get_provider_info(&param.addr) {
+        match provider.status {
+            provider::Status::OK | provider::Status::PREPARATION => {
+                let imap = provider.get_imap_server();
+                let smtp = provider.get_smtp_server();
+                // clippy complains about these is_some()/unwrap() settings,
+                // however, rewriting the code to "if let" would make things less obvious,
+                // esp. if we allow more combinations of servers (pop, jmap).
+                // therefore, #[allow(clippy::unnecessary_unwrap)] is added above.
+                if imap.is_some() && smtp.is_some() {
+                    let imap = imap.unwrap();
+                    let smtp = smtp.unwrap();
+
+                    let mut p = LoginParam::new();
+                    p.addr = param.addr.clone();
+
+                    p.mail_server = imap.hostname.to_string();
+                    p.mail_user = imap.apply_username_pattern(param.addr.clone());
+                    p.mail_port = imap.port as i32;
+                    p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                    p.server_flags |= match imap.socket {
+                        provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
+                        provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
+                    };
+
+                    p.send_server = smtp.hostname.to_string();
+                    p.send_user = smtp.apply_username_pattern(param.addr.clone());
+                    p.send_port = smtp.port as i32;
+                    p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                    p.server_flags |= match smtp.socket {
+                        provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
+                        provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
+                    };
+
+                    info!(context, "offline autoconfig found: {}", p);
+                    return Some(p);
+                } else {
+                    info!(context, "offline autoconfig found, but no servers defined");
+                    return None;
+                }
+            }
+            provider::Status::BROKEN => {
+                info!(context, "offline autoconfig found, provider is broken");
+                return None;
+            }
+        }
+    }
+    info!(context, "no offline autoconfig found");
+    None
 }
 
 fn try_imap_connections(
@@ -485,17 +568,22 @@ fn try_imap_connection(
 
 fn try_imap_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
     let inf = format!(
-        "imap: {}@{}:{} flags=0x{:x}",
-        param.mail_user, param.mail_server, param.mail_port, param.server_flags
+        "imap: {}@{}:{} flags=0x{:x} certificate_checks={}",
+        param.mail_user,
+        param.mail_server,
+        param.mail_port,
+        param.server_flags,
+        param.imap_certificate_checks
     );
     info!(context, "Trying: {}", inf);
-    if context
-        .inbox_thread
-        .read()
-        .unwrap()
-        .imap
-        .connect(context, &param)
-    {
+    if task::block_on(
+        context
+            .inbox_thread
+            .read()
+            .unwrap()
+            .imap
+            .connect(context, &param),
+    ) {
         info!(context, "success: {}", inf);
         return Some(true);
     }
@@ -564,9 +652,32 @@ fn try_smtp_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("Invalid email address: {0:?}")]
+    InvalidEmailAddress(String),
+
+    #[error("XML error at position {position}")]
+    InvalidXml {
+        position: usize,
+        #[source]
+        error: quick_xml::Error,
+    },
+
+    #[error("Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[error("Failed to get URL")]
+    ReadUrlError(#[from] self::read_url::Error),
+
+    #[error("Number of redirection is exceeded")]
+    RedirectionError,
+}
+
 #[cfg(test)]
 mod tests {
 
+    use super::*;
     use crate::config::*;
     use crate::configure::JobConfigureImap;
     use crate::test_utils::*;
@@ -580,4 +691,19 @@ mod tests {
         t.ctx.set_config(Config::MailPw, Some("123456")).unwrap();
         JobConfigureImap(&t.ctx);
     }
+
+    #[test]
+    fn test_get_offline_autoconfig() {
+        let context = dummy_context().ctx;
+
+        let mut params = LoginParam::new();
+        params.addr = "someone123@example.org".to_string();
+        assert!(get_offline_autoconfig(&context, &params).is_none());
+
+        let mut params = LoginParam::new();
+        params.addr = "someone123@nauta.cu".to_string();
+        let found_params = get_offline_autoconfig(&context, &params).unwrap();
+        assert_eq!(found_params.mail_server, "imap.nauta.cu".to_string());
+        assert_eq!(found_params.send_server, "smtp.nauta.cu".to_string());
+    }
 }

src/configure/read_url.rs

@@ -1,21 +1,18 @@
 use crate::context::Context;
-use failure::Fail;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "URL request error")]
-    GetError(#[cause] reqwest::Error),
+    #[error("URL request error")]
+    GetError(#[from] reqwest::Error),
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
-
-pub fn read_url(context: &Context, url: &str) -> Result<String> {
+pub fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     info!(context, "Requesting URL {}", url);
 
-    match reqwest::Client::new()
+    match reqwest::blocking::Client::new()
         .get(url)
         .send()
-        .and_then(|mut res| res.text())
+        .and_then(|res| res.text())
     {
         Ok(res) => Ok(res),
         Err(err) => {

src/constants.rs

@@ -1,8 +1,9 @@
 //! # Constants
-#![allow(non_camel_case_types, dead_code)]
+#![allow(dead_code)]
 
 use deltachat_derive::*;
 use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
 
 lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
@@ -15,7 +16,20 @@ const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug,
+    Display,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
+    Serialize,
+    Deserialize,
+)]
 #[repr(u8)]
 pub enum Blocked {
     Not = 0,
@@ -43,24 +57,38 @@ impl Default for ShowEmails {
     }
 }
 
-pub const DC_IMAP_SEEN: u32 = 0x1;
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum KeyGenType {
+    Default = 0,
+    Rsa2048 = 1,
+    Ed25519 = 2,
+}
+
+impl Default for KeyGenType {
+    fn default() -> Self {
+        KeyGenType::Default
+    }
+}
 
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
 
+pub(crate) const DC_FROM_HANDSHAKE: i32 = 0x01;
+
 pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
+pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
 
 pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
 pub const DC_GCL_ADD_SELF: usize = 0x02;
 
-// values for DC_PARAM_FORCE_PLAINTEXT
-pub(crate) const DC_FP_NO_AUTOCRYPT_HEADER: i32 = 2;
-pub(crate) const DC_FP_ADD_AUTOCRYPT_HEADER: i32 = 1;
+// unchanged user avatars are resent to the recipients every some days
+pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
@@ -89,6 +117,8 @@ pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
     FromSql,
     ToSql,
     IntoStaticStr,
+    Serialize,
+    Deserialize,
 )]
 #[repr(u32)]
 pub enum Chattype {
@@ -111,7 +141,7 @@ pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 /// approx. max. length returned by dc_msg_get_text()
 const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100000;
+const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -119,6 +149,10 @@ pub const DC_CONTACT_ID_INFO: u32 = 2;
 pub const DC_CONTACT_ID_DEVICE: u32 = 5;
 pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
 
+// decorative address that is used for DC_CONTACT_ID_DEVICE
+// when an api that returns an email is called.
+pub const DC_CONTACT_ID_DEVICE_ADDR: &str = "device@localhost";
+
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;
@@ -163,13 +197,13 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
 pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
 
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
+pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN);
+    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN);
+    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
 
 // QR code scanning (view from Bob, the joiner)
 pub const DC_VC_AUTH_REQUIRED: i32 = 2;
@@ -177,10 +211,30 @@ pub const DC_VC_CONTACT_CONFIRM: i32 = 6;
 pub const DC_BOB_ERROR: i32 = 0;
 pub const DC_BOB_SUCCESS: i32 = 1;
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+// max. width/height of an avatar
+pub const AVATAR_SIZE: u32 = 192;
+
+// this value can be increased if the folder configuration is changed and must be redone on next program start
+pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
+
+#[derive(
+    Debug,
+    Display,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
+    Serialize,
+    Deserialize,
+)]
 #[repr(i32)]
 pub enum Viewtype {
     Unknown = 0,
+
     /// Text message.
     /// The text of the message is set using dc_msg_set_text()
     /// and retrieved with dc_msg_get_text().
@@ -260,8 +314,6 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
 const DC_STR_DRAFT: usize = 3;
-const DC_STR_MEMBER: usize = 4;
-const DC_STR_CONTACT: usize = 6;
 const DC_STR_VOICEMESSAGE: usize = 7;
 const DC_STR_DEADDROP: usize = 8;
 const DC_STR_IMAGE: usize = 9;
@@ -293,7 +345,6 @@ const DC_STR_ARCHIVEDCHATS: usize = 40;
 const DC_STR_STARREDMSGS: usize = 41;
 const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
 const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_SELFTALK_SUBTITLE: usize = 50;
 const DC_STR_CANNOT_LOGIN: usize = 60;
 const DC_STR_SERVER_RESPONSE: usize = 61;
 const DC_STR_MSGACTIONBYUSER: usize = 62;

src/contact.rs

@@ -1,37 +1,39 @@
+//! Contacts module
+
 use std::path::PathBuf;
 
 use deltachat_derive::*;
 use itertools::Itertools;
-use rusqlite;
 
 use crate::aheader::EncryptPreference;
+use crate::chat::ChatId;
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::e2ee;
-use crate::error::Result;
+use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
-use crate::key::*;
+use crate::key::{DcKey, Key, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
+use crate::mimeparser::AvatarAction;
+use crate::param::*;
 use crate::peerstate::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-/// Contacts with at least this origin value are shown in the contact list.
-const DC_ORIGIN_MIN_CONTACT_LIST: i32 = 0x100;
-
 /// An object representing a single contact in memory.
+///
 /// The contact object is not updated.
 /// If you want an update, you have to recreate the object.
 ///
 /// The library makes sure
 /// only to use names _authorized_ by the contact in `To:` or `Cc:`.
-/// _Given-names _as "Daddy" or "Honey" are not used there.
+/// *Given-names* as "Daddy" or "Honey" are not used there.
 /// For this purpose, internally, two names are tracked -
-/// authorized-name and given-name.
+/// authorized name and given name.
 /// By default, these names are equal, but functions working with contact names
+/// only affect the given name.
 #[derive(Debug)]
 pub struct Contact {
     /// The contact ID.
@@ -42,20 +44,28 @@ pub struct Contact {
     ///
     /// Normal contact IDs are larger than these special ones (larger than DC_CONTACT_ID_LAST_SPECIAL).
     pub id: u32,
+
     /// Contact name. It is recommended to use `Contact::get_name`,
     /// `Contact::get_display_name` or `Contact::get_name_n_addr` to access this field.
     /// May be empty, initially set to `authname`.
     name: String,
+
     /// Name authorized by the contact himself. Only this name may be spread to others,
     /// e.g. in To:-lists. May be empty. It is recommended to use `Contact::get_authname`,
     /// to access this field.
     authname: String,
-    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr`` to access this field.
+
+    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr` to access this field.
     addr: String,
+
     /// Blocked state. Use dc_contact_is_blocked to access this field.
-    blocked: bool,
+    pub blocked: bool,
+
     /// The origin/source of the contact.
     pub origin: Origin,
+
+    /// Parameters as Param::ProfileImage
+    pub param: Params,
 }
 
 /// Possible origins of a contact.
@@ -65,38 +75,55 @@ pub struct Contact {
 #[repr(i32)]
 pub enum Origin {
     Unknown = 0,
+
     /// From: of incoming messages of unknown sender
     IncomingUnknownFrom = 0x10,
+
     /// Cc: of incoming messages of unknown sender
     IncomingUnknownCc = 0x20,
+
     /// To: of incoming messages of unknown sender
     IncomingUnknownTo = 0x40,
+
     /// address scanned but not verified
     UnhandledQrScan = 0x80,
+
     /// Reply-To: of incoming message of known sender
+    /// Contacts with at least this origin value are shown in the contact list.
     IncomingReplyTo = 0x100,
+
     /// Cc: of incoming message of known sender
     IncomingCc = 0x200,
+
     /// additional To:'s of incoming message of known sender
     IncomingTo = 0x400,
+
     /// a chat was manually created for this user, but no message yet sent
     CreateChat = 0x800,
+
     /// message sent by us
     OutgoingBcc = 0x1000,
+
     /// message sent by us
     OutgoingCc = 0x2000,
+
     /// message sent by us
     OutgoingTo = 0x4000,
+
     /// internal use
     Internal = 0x40000,
+
     /// address is in our address book
-    AdressBook = 0x80000,
+    AddressBook = 0x80000,
+
     /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinInvited = 0x1000000,
+    SecurejoinInvited = 0x0100_0000,
+
     /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinJoined = 0x2000000,
-    /// contact added mannually by dc_create_contact(), this should be the largets origin as otherwise the user cannot modify the names
-    ManuallyCreated = 0x4000000,
+    SecurejoinJoined = 0x0200_0000,
+
+    /// contact added mannually by dc_create_contact(), this should be the largest origin as otherwise the user cannot modify the names
+    ManuallyCreated = 0x0400_0000,
 }
 
 impl Default for Origin {
@@ -106,24 +133,16 @@ impl Default for Origin {
 }
 
 impl Origin {
-    /// Contacts that start a new "normal" chat, defaults to off.
-    pub fn is_start_new_chat(self) -> bool {
-        self as i32 >= 0x7FFFFFFF
-    }
-
-    /// Contacts that are verified and known not to be spam.
-    pub fn is_verified(self) -> bool {
-        self as i32 >= 0x100
-    }
-
-    /// Contacts that are shown in the contact list.
-    pub fn include_in_contactlist(self) -> bool {
-        self as i32 >= DC_ORIGIN_MIN_CONTACT_LIST
+    /// Contacts that are known, i. e. they came in via accepted contacts or
+    /// themselves an accepted contact. Known contacts are shown in the
+    /// contact list when one creates a chat and wants to add members etc.
+    pub fn is_known(self) -> bool {
+        self >= Origin::IncomingReplyTo
     }
 }
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub enum Modifier {
+pub(crate) enum Modifier {
     None,
     Modified,
     Created,
@@ -142,32 +161,10 @@ pub enum VerifiedStatus {
 
 impl Contact {
     pub fn load_from_db(context: &Context, contact_id: u32) -> crate::sql::Result<Self> {
-        if contact_id == DC_CONTACT_ID_SELF {
-            let contact = Contact {
-                id: contact_id,
-                name: context.stock_str(StockMessage::SelfMsg).into(),
-                authname: "".into(),
-                addr: context
-                    .get_config(Config::ConfiguredAddr)
-                    .unwrap_or_default(),
-                blocked: false,
-                origin: Origin::Unknown,
-            };
-            return Ok(contact);
-        } else if contact_id == DC_CONTACT_ID_DEVICE {
-            let contact = Contact {
-                id: contact_id,
-                name: context.stock_str(StockMessage::DeviceMessages).into(),
-                authname: "".into(),
-                addr: "device@localhost".into(),
-                blocked: false,
-                origin: Origin::Unknown,
-            };
-            return Ok(contact);
-        }
-
-        context.sql.query_row(
-            "SELECT c.name, c.addr, c.origin, c.blocked, c.authname  FROM contacts c  WHERE c.id=?;",
+        let mut res = context.sql.query_row(
+            "SELECT c.name, c.addr, c.origin, c.blocked, c.authname, c.param
+               FROM contacts c
+              WHERE c.id=?;",
             params![contact_id as i32],
             |row| {
                 let contact = Self {
@@ -177,10 +174,21 @@ impl Contact {
                     addr: row.get::<_, String>(1)?,
                     blocked: row.get::<_, Option<i32>>(3)?.unwrap_or_default() != 0,
                     origin: row.get(2)?,
+                    param: row.get::<_, String>(5)?.parse().unwrap_or_default(),
                 };
                 Ok(contact)
-            }
-        )
+            },
+        )?;
+        if contact_id == DC_CONTACT_ID_SELF {
+            res.name = context.stock_str(StockMessage::SelfMsg).to_string();
+            res.addr = context
+                .get_config(Config::ConfiguredAddr)
+                .unwrap_or_default();
+        } else if contact_id == DC_CONTACT_ID_DEVICE {
+            res.name = context.stock_str(StockMessage::DeviceMessages).to_string();
+            res.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
+        }
+        Ok(res)
     }
 
     /// Returns `true` if this contact is blocked.
@@ -208,7 +216,7 @@ impl Contact {
     /// Add a single contact as a result of an _explicit_ user action.
     ///
     /// We assume, the contact name, if any, is entered by the user and is used "as is" therefore,
-    /// normalize() is _not_ called for the name. If the contact is blocked, it is unblocked.
+    /// normalize() is *not* called for the name. If the contact is blocked, it is unblocked.
     ///
     /// To add a number of contacts, see `dc_add_address_book()` which is much faster for adding
     /// a bunch of addresses.
@@ -238,7 +246,7 @@ impl Contact {
     }
 
     /// Mark all messages sent by the given contact
-    /// as _noticed_.  See also dc_marknoticed_chat() and dc_markseen_msgs()
+    /// as *noticed*.  See also dc_marknoticed_chat() and dc_markseen_msgs()
     ///
     /// Calling this function usually results in the event `#DC_EVENT_MSGS_CHANGED`.
     pub fn mark_noticed(context: &Context, id: u32) {
@@ -251,7 +259,7 @@ impl Contact {
         .is_ok()
         {
             context.call_cb(Event::MsgsChanged {
-                chat_id: 0,
+                chat_id: ChatId::new(0),
                 msg_id: MsgId::new(0),
             });
         }
@@ -262,7 +270,7 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `dc_may_be_valid_addr()`.
-    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>) -> u32 {
+    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>, min_origin: Origin) -> u32 {
         if addr.as_ref().is_empty() {
             return 0;
         }
@@ -275,22 +283,43 @@ impl Contact {
         if addr_cmp(addr_normalized, addr_self) {
             return DC_CONTACT_ID_SELF;
         }
-
         context.sql.query_get_value(
             context,
             "SELECT id FROM contacts WHERE addr=?1 COLLATE NOCASE AND id>?2 AND origin>=?3 AND blocked=0;",
             params![
                 addr_normalized,
                 DC_CONTACT_ID_LAST_SPECIAL as i32,
-                DC_ORIGIN_MIN_CONTACT_LIST,
+                min_origin as u32,
             ],
         ).unwrap_or_default()
     }
 
     /// Lookup a contact and create it if it does not exist yet.
+    /// The contact is identified by the email-address, a name and an "origin" can be given.
+    ///
+    /// The "origin" is where the address comes from -
+    /// from-header, cc-header, addressbook, qr, manual-edit etc.
+    /// In general, "better" origins overwrite the names of "worse" origins -
+    /// Eg. if we got a name in cc-header and later in from-header, the name will change -
+    /// this does not happen the other way round.
+    ///
+    /// The "best" origin are manually created contacts -
+    /// names given manually can only be overwritten by further manual edits
+    /// (until they are set empty again or reset to the name seen in the From-header).
+    ///
+    /// These manually edited names are _never_ used for sending on the wire -
+    /// this should avoid sending sth. as "Mama" or "Daddy" to some 3rd party.
+    /// Instead, for the wire, we use so called "authnames"
+    /// that can only be set and updated by a From-header.
+    ///
+    /// The different names used in the function are:
+    /// - "name": name passed as function argument, belonging to the given origin
+    /// - "row_name": current name used in the database, typically set to "name"
+    /// - "row_authname": name as authorized from a contact, set only through a From-header
+    /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
     ///
     /// Returns the contact_id and a `Modifier` value indicating if a modification occured.
-    pub fn add_or_lookup(
+    pub(crate) fn add_or_lookup(
         context: &Context,
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
@@ -344,7 +373,9 @@ impl Contact {
 
                 if !name.as_ref().is_empty() {
                     if !row_name.is_empty() {
-                        if origin >= row_origin && name.as_ref() != row_name {
+                        if (origin >= row_origin || row_name == row_authname)
+                            && name.as_ref() != row_name
+                        {
                             update_name = true;
                         }
                     } else {
@@ -353,6 +384,9 @@ impl Contact {
                     if origin == Origin::IncomingUnknownFrom && name.as_ref() != row_authname {
                         update_authname = true;
                     }
+                } else if origin == Origin::ManuallyCreated && !row_authname.is_empty() {
+                    // no name given on manual edit, this will update the name to the authname
+                    update_name = true;
                 }
 
                 Ok((row_id, row_name, row_addr, row_origin, row_authname))
@@ -363,16 +397,22 @@ impl Contact {
                 update_addr = true;
             }
             if update_name || update_authname || update_addr || origin > row_origin {
+                let new_name = if update_name {
+                    if !name.as_ref().is_empty() {
+                        name.as_ref()
+                    } else {
+                        &row_authname
+                    }
+                } else {
+                    &row_name
+                };
+
                 sql::execute(
                     context,
                     &context.sql,
                     "UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
                     params![
-                        if update_name {
-                            name.as_ref()
-                        } else {
-                            &row_name
-                        },
+                        new_name,
                         if update_addr { addr } else { &row_addr },
                         if origin > row_origin {
                             origin
@@ -390,27 +430,41 @@ impl Contact {
                 .ok();
 
                 if update_name {
+                    // Update the contact name also if it is used as a group name.
+                    // This is one of the few duplicated data, however, getting the chat list is easier this way.
                     sql::execute(
                     context,
                     &context.sql,
                     "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    params![name.as_ref(), Chattype::Single, row_id]
+                    params![new_name, Chattype::Single, row_id]
                 ).ok();
                 }
                 sth_modified = Modifier::Modified;
             }
-        } else if sql::execute(
-            context,
-            &context.sql,
-            "INSERT INTO contacts (name, addr, origin) VALUES(?, ?, ?);",
-            params![name.as_ref(), addr, origin,],
-        )
-        .is_ok()
-        {
-            row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
-            sth_modified = Modifier::Created;
         } else {
-            error!(context, "Cannot add contact.");
+            if origin == Origin::IncomingUnknownFrom {
+                update_authname = true;
+            }
+
+            if sql::execute(
+                context,
+                &context.sql,
+                "INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
+                params![
+                    name.as_ref(),
+                    addr,
+                    origin,
+                    if update_authname { name.as_ref() } else { "" }
+                ],
+            )
+            .is_ok()
+            {
+                row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
+                sth_modified = Modifier::Created;
+                info!(context, "added contact id={} addr={}", row_id, addr);
+            } else {
+                error!(context, "Cannot add contact.");
+            }
         }
 
         Ok((row_id, sth_modified))
@@ -428,7 +482,7 @@ impl Contact {
     /// the event `DC_EVENT_CONTACTS_CHANGED` is sent.
     ///
     /// To add a single contact entered by the user, you should prefer `Contact::create`,
-    /// however, for adding a bunch of addresses, this function is _much_ faster.
+    /// however, for adding a bunch of addresses, this function is much faster.
     ///
     /// The `addr_book` is a multiline string in the format `Name one\nAddress one\nName two\nAddress two`.
     ///
@@ -438,9 +492,18 @@ impl Contact {
 
         for (name, addr) in split_address_book(addr_book.as_ref()).into_iter() {
             let name = normalize_name(name);
-            let (_, modified) = Contact::add_or_lookup(context, name, addr, Origin::AdressBook)?;
-            if modified != Modifier::None {
-                modify_cnt += 1
+            match Contact::add_or_lookup(context, name, addr, Origin::AddressBook) {
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Failed to add address {} from address book: {}", addr, err
+                    );
+                }
+                Ok((_, modified)) => {
+                    if modified != Modifier::None {
+                        modify_cnt += 1
+                    }
+                }
             }
         }
         if modify_cnt > 0 {
@@ -494,7 +557,7 @@ impl Contact {
                 params![
                     self_addr,
                     DC_CONTACT_ID_LAST_SPECIAL as i32,
-                    0x100,
+                    Origin::IncomingReplyTo,
                     &s3str_like_cmd,
                     &s3str_like_cmd,
                     if flag_verified_only { 0 } else { 1 },
@@ -583,8 +646,6 @@ impl Contact {
             let peerstate = Peerstate::from_addr(context, &context.sql, &contact.addr);
             let loginparam = LoginParam::from_database(context, "configured_");
 
-            let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
-
             if peerstate.is_some()
                 && peerstate
                     .as_ref()
@@ -599,16 +660,11 @@ impl Contact {
                         StockMessage::E2eAvailable
                     });
                 ret += &p;
-                if self_key.is_none() {
-                    e2ee::ensure_secret_key_exists(context)?;
-                    self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
-                }
+                let self_key = Key::from(SignedPublicKey::load_self(context)?);
                 let p = context.stock_str(StockMessage::FingerPrints);
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key
-                    .map(|k| k.formatted_fingerprint())
-                    .unwrap_or_default();
+                let fingerprint_self = self_key.formatted_fingerprint();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
                     .map(|k| k.formatted_fingerprint())
@@ -712,6 +768,16 @@ impl Contact {
         Ok(Contact::load_from_db(context, contact_id)?)
     }
 
+    pub fn update_param(&mut self, context: &Context) -> Result<()> {
+        sql::execute(
+            context,
+            &context.sql,
+            "UPDATE contacts SET param=? WHERE id=?",
+            params![self.param.to_string(), self.id as i32],
+        )?;
+        Ok(())
+    }
+
     /// Get the ID of the contact.
     pub fn get_id(&self) -> u32 {
         self.id
@@ -745,6 +811,9 @@ impl Contact {
         if !self.name.is_empty() {
             return &self.name;
         }
+        if !self.authname.is_empty() {
+            return &self.authname;
+        }
         &self.addr
     }
 
@@ -780,8 +849,11 @@ impl Contact {
             if let Some(p) = context.get_config(Config::Selfavatar) {
                 return Some(PathBuf::from(p));
             }
+        } else if let Some(image_rel) = self.param.get(Param::ProfileImage) {
+            if !image_rel.is_empty() {
+                return Some(dc_get_abs_path(context, image_rel));
+            }
         }
-        // TODO: else get image_abs from contact param
         None
     }
 
@@ -864,22 +936,6 @@ impl Contact {
             .unwrap_or_default() as usize
     }
 
-    pub fn get_origin_by_id(context: &Context, contact_id: u32, ret_blocked: &mut i32) -> Origin {
-        let mut ret = Origin::Unknown;
-        *ret_blocked = 0;
-
-        if let Ok(contact) = Contact::load_from_db(context, contact_id) {
-            /* we could optimize this by loading only the needed fields */
-            if contact.blocked {
-                *ret_blocked = 1;
-            } else {
-                ret = contact.origin;
-            }
-        }
-
-        ret
-    }
-
     pub fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
         if !context.sql.is_open() || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
             return false;
@@ -906,7 +962,7 @@ impl Contact {
 }
 
 /// Extracts first name from full name.
-fn get_first_name<'a>(full_name: &'a str) -> &'a str {
+fn get_first_name(full_name: &str) -> &str {
     full_name.splitn(2, ' ').next().unwrap_or_default()
 }
 
@@ -933,21 +989,21 @@ fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
     }
 
     if let Ok(contact) = Contact::load_from_db(context, contact_id) {
-        if contact.blocked != new_blocking {
-            if sql::execute(
+        if contact.blocked != new_blocking
+            && sql::execute(
                 context,
                 &context.sql,
                 "UPDATE contacts SET blocked=? WHERE id=?;",
                 params![new_blocking as i32, contact_id as i32],
             )
             .is_ok()
-            {
-                // also (un)block all chats with _only_ this contact - we do not delete them to allow a
-                // non-destructive blocking->unblocking.
-                // (Maybe, beside normal chats (type=100) we should also block group chats with only this user.
-                // However, I'm not sure about this point; it may be confusing if the user wants to add other people;
-                // this would result in recreating the same group...)
-                if sql::execute(
+        {
+            // also (un)block all chats with _only_ this contact - we do not delete them to allow a
+            // non-destructive blocking->unblocking.
+            // (Maybe, beside normal chats (type=100) we should also block group chats with only this user.
+            // However, I'm not sure about this point; it may be confusing if the user wants to add other people;
+            // this would result in recreating the same group...)
+            if sql::execute(
                     context,
                     &context.sql,
                     "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
@@ -956,11 +1012,35 @@ fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
                     Contact::mark_noticed(context, contact_id);
                     context.call_cb(Event::ContactsChanged(None));
                 }
-            }
         }
     }
 }
 
+pub(crate) fn set_profile_image(
+    context: &Context,
+    contact_id: u32,
+    profile_image: &AvatarAction,
+) -> Result<()> {
+    // the given profile image is expected to be already in the blob directory
+    // as profile images can be set only by receiving messages, this should be always the case, however.
+    let mut contact = Contact::load_from_db(context, contact_id)?;
+    let changed = match profile_image {
+        AvatarAction::Change(profile_image) => {
+            contact.param.set(Param::ProfileImage, profile_image);
+            true
+        }
+        AvatarAction::Delete => {
+            contact.param.remove(Param::ProfileImage);
+            true
+        }
+    };
+    if changed {
+        contact.update_param(context)?;
+        context.call_cb(Event::ContactsChanged(Some(contact_id)));
+    }
+    Ok(())
+}
+
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
@@ -975,7 +1055,7 @@ pub fn normalize_name(full_name: impl AsRef<str>) -> String {
     }
 
     let len = full_name.len();
-    if len > 0 {
+    if len > 1 {
         let firstchar = full_name.as_bytes()[0];
         let lastchar = full_name.as_bytes()[len - 1];
         if firstchar == b'\'' && lastchar == b'\''
@@ -1025,6 +1105,17 @@ fn cat_fingerprint(
     }
 }
 
+impl Context {
+    /// determine whether the specified addr maps to the/a self addr
+    pub fn is_self_addr(&self, addr: &str) -> Result<bool> {
+        let self_addr = self
+            .get_config(Config::ConfiguredAddr)
+            .ok_or_else(|| format_err!("Not configured"))?;
+
+        Ok(addr_cmp(self_addr, addr))
+    }
+}
+
 pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
     let norm1 = addr_normalize(addr1.as_ref()).to_lowercase();
     let norm2 = addr_normalize(addr2.as_ref()).to_lowercase();
@@ -1032,15 +1123,6 @@ pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
     norm1 == norm2
 }
 
-pub fn addr_equals_self(context: &Context, addr: impl AsRef<str>) -> bool {
-    if !addr.as_ref().is_empty() {
-        if let Some(self_addr) = context.get_config(Config::ConfiguredAddr) {
-            return addr_cmp(addr, self_addr);
-        }
-    }
-    false
-}
-
 fn split_address_book(book: &str) -> Vec<(&str, &str)> {
     book.lines()
         .chunks(2)
@@ -1081,6 +1163,10 @@ mod tests {
     fn test_normalize_name() {
         assert_eq!(&normalize_name("Doe, John"), "John Doe");
         assert_eq!(&normalize_name(" hello world   "), "hello world");
+        assert_eq!(&normalize_name("<"), "<");
+        assert_eq!(&normalize_name(">"), ">");
+        assert_eq!(&normalize_name("'"), "'");
+        assert_eq!(&normalize_name("\""), "\"");
     }
 
     #[test]
@@ -1124,6 +1210,18 @@ mod tests {
         assert_eq!(contacts.len(), 0);
     }
 
+    #[test]
+    fn test_is_self_addr() -> Result<()> {
+        let t = test_context(None);
+        assert!(t.ctx.is_self_addr("me@me.org").is_err());
+
+        let addr = configure_alice_keypair(&t.ctx);
+        assert_eq!(t.ctx.is_self_addr("me@me.org")?, false);
+        assert_eq!(t.ctx.is_self_addr(&addr)?, true);
+
+        Ok(())
+    }
+
     #[test]
     fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
@@ -1131,6 +1229,7 @@ mod tests {
         let book = concat!(
             "  Name one  \n one@eins.org \n",
             "Name two\ntwo@deux.net\n",
+            "Invalid\n+1234567890\n", // invalid, should be ignored
             "\nthree@drei.sam\n",
             "Name two\ntwo@deux.net\n" // should not be added again
         );
@@ -1213,6 +1312,139 @@ mod tests {
         assert!(!contact.is_blocked());
     }
 
+    #[test]
+    fn test_remote_authnames() {
+        let t = dummy_context();
+
+        // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
+        let (contact_id, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "bob1",
+            "bob@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Created);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob1");
+        assert_eq!(contact.get_name(), "bob1");
+        assert_eq!(contact.get_display_name(), "bob1");
+
+        // incoming mail `From: bob2 <bob@example.org>` - this should update authname and name
+        let (contact_id, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "bob2",
+            "bob@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob2");
+        assert_eq!(contact.get_name(), "bob2");
+        assert_eq!(contact.get_display_name(), "bob2");
+
+        // manually edit name to "bob3" - authname should be still be "bob2" a given in `From:` above
+        let contact_id = Contact::create(&t.ctx, "bob3", "bob@example.org").unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob2");
+        assert_eq!(contact.get_name(), "bob3");
+        assert_eq!(contact.get_display_name(), "bob3");
+
+        // incoming mail `From: bob4 <bob@example.org>` - this should update authname, manually given name is still "bob3"
+        let (contact_id, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "bob4",
+            "bob@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob4");
+        assert_eq!(contact.get_name(), "bob3");
+        assert_eq!(contact.get_display_name(), "bob3");
+    }
+
+    #[test]
+    fn test_remote_authnames_create_empty() {
+        let t = dummy_context();
+
+        // manually create "claire@example.org" without a given name
+        let contact_id = Contact::create(&t.ctx, "", "claire@example.org").unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "");
+        assert_eq!(contact.get_name(), "");
+        assert_eq!(contact.get_display_name(), "claire@example.org");
+
+        // incoming mail `From: claire1 <claire@example.org>` - this should update authname and name
+        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "claire1",
+            "claire@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_same);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "claire1");
+        assert_eq!(contact.get_name(), "claire1");
+        assert_eq!(contact.get_display_name(), "claire1");
+
+        // incoming mail `From: claire2 <claire@example.org>` - this should update authname and name
+        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "claire2",
+            "claire@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_same);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "claire2");
+        assert_eq!(contact.get_name(), "claire2");
+        assert_eq!(contact.get_display_name(), "claire2");
+    }
+
+    #[test]
+    fn test_remote_authnames_edit_empty() {
+        let t = dummy_context();
+
+        // manually create "dave@example.org"
+        let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org").unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "");
+        assert_eq!(contact.get_name(), "dave1");
+        assert_eq!(contact.get_display_name(), "dave1");
+
+        // incoming mail `From: dave2 <dave@example.org>` - this should update authname
+        Contact::add_or_lookup(
+            &t.ctx,
+            "dave2",
+            "dave@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "dave2");
+        assert_eq!(contact.get_name(), "dave1");
+        assert_eq!(contact.get_display_name(), "dave1");
+
+        // manually clear the name
+        Contact::create(&t.ctx, "", "dave@example.org").unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "dave2");
+        assert_eq!(contact.get_name(), "dave2");
+        assert_eq!(contact.get_display_name(), "dave2");
+    }
+
     #[test]
     fn test_addr_cmp() {
         assert!(addr_cmp("AA@AA.ORG", "aa@aa.ORG"));

src/context.rs

@@ -1,26 +1,28 @@
-use std::collections::HashMap;
+//! Context module
+
+use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Condvar, Mutex, RwLock};
 
-use libc::uintptr_t;
-
 use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
+use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::Event;
 use crate::imap::*;
 use crate::job::*;
 use crate::job_thread::JobThread;
-use crate::key::*;
+use crate::key::{DcKey, Key, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message, MsgId};
+use crate::message::{self, Message, MessengerMessage, MsgId};
 use crate::param::Params;
 use crate::smtp::Smtp;
 use crate::sql::Sql;
+use std::time::SystemTime;
 
 /// Callback function type for [Context]
 ///
@@ -30,12 +32,7 @@ use crate::sql::Sql;
 /// * `event` - One of the [Event] items.
 /// * `data1` - Depends on the event parameter, see [Event].
 /// * `data2` - Depends on the event parameter, see [Event].
-///
-/// # Returns
-///
-/// This callback must return 0 unless stated otherwise in the event
-/// description at [Event].
-pub type ContextCallback = dyn Fn(&Context, Event) -> uintptr_t + Send + Sync;
+pub type ContextCallback = dyn Fn(&Context, Event) -> () + Send + Sync;
 
 #[derive(DebugStub)]
 pub struct Context {
@@ -55,13 +52,14 @@ pub struct Context {
     #[debug_stub = "Callback"]
     cb: Box<ContextCallback>,
     pub os_name: Option<String>,
-    pub cmdline_sel_chat_id: Arc<RwLock<u32>>,
-    pub bob: Arc<RwLock<BobStatus>>,
+    pub cmdline_sel_chat_id: Arc<RwLock<ChatId>>,
+    pub(crate) bob: Arc<RwLock<BobStatus>>,
     pub last_smeared_timestamp: RwLock<i64>,
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
     pub translated_stockstrings: RwLock<HashMap<usize, String>>,
+    creation_time: SystemTime,
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -76,20 +74,11 @@ pub struct RunningState {
 /// actual keys and their values which will be present are not
 /// guaranteed.  Calling [Context::get_info] also includes information
 /// about the context on top of the information here.
-pub fn get_info() -> HashMap<&'static str, String> {
-    let mut res = HashMap::new();
+pub fn get_info() -> BTreeMap<&'static str, String> {
+    let mut res = BTreeMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
-    res.insert(
-        "sqlite_thread_safe",
-        unsafe { rusqlite::ffi::sqlite3_threadsafe() }.to_string(),
-    );
-    res.insert(
-        "arch",
-        (std::mem::size_of::<*mut libc::c_void>())
-            .wrapping_mul(8)
-            .to_string(),
-    );
+    res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
     res.insert("level", "awesome".into());
     res
 }
@@ -97,6 +86,8 @@ pub fn get_info() -> HashMap<&'static str, String> {
 impl Context {
     /// Creates new context.
     pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
+        pretty_env_logger::try_init_timed().ok();
+
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
@@ -130,7 +121,7 @@ impl Context {
             oauth2_critical: Arc::new(Mutex::new(())),
             bob: Arc::new(RwLock::new(Default::default())),
             last_smeared_timestamp: RwLock::new(0),
-            cmdline_sel_chat_id: Arc::new(RwLock::new(0)),
+            cmdline_sel_chat_id: Arc::new(RwLock::new(ChatId::new(0))),
             inbox_thread: Arc::new(RwLock::new(JobThread::new(
                 "INBOX",
                 "configured_inbox_folder",
@@ -150,6 +141,7 @@ impl Context {
             perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
+            creation_time: std::time::SystemTime::now(),
         };
 
         ensure!(
@@ -170,8 +162,8 @@ impl Context {
         self.blobdir.as_path()
     }
 
-    pub fn call_cb(&self, event: Event) -> uintptr_t {
-        (*self.cb)(self, event)
+    pub fn call_cb(&self, event: Event) {
+        (*self.cb)(self, event);
     }
 
     /*******************************************************************************
@@ -234,7 +226,7 @@ impl Context {
      * UI chat/message related API
      ******************************************************************************/
 
-    pub fn get_info(&self) -> HashMap<&'static str, String> {
+    pub fn get_info(&self) -> BTreeMap<&'static str, String> {
         let unset = "0";
         let l = LoginParam::from_database(self, "");
         let l2 = LoginParam::from_database(self, "configured_");
@@ -263,10 +255,9 @@ impl Context {
             rusqlite::NO_PARAMS,
         );
 
-        let fingerprint_str = if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql) {
-            key.fingerprint()
-        } else {
-            "<Not yet calculated>".into()
+        let fingerprint_str = match SignedPublicKey::load_self(self) {
+            Ok(key) => Key::from(key).fingerprint(),
+            Err(err) => format!("<key failure: {}>", err),
         };
 
         let inbox_watch = self.get_config_int(Config::InboxWatch);
@@ -296,6 +287,11 @@ impl Context {
         res.insert("database_version", dbversion.to_string());
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
+        res.insert(
+            "selfavatar",
+            self.get_config(Config::Selfavatar)
+                .unwrap_or_else(|| "<unset>".to_string()),
+        );
         res.insert("is_configured", is_configured.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
@@ -319,6 +315,9 @@ impl Context {
         );
         res.insert("fingerprint", fingerprint_str);
 
+        let elapsed = self.creation_time.elapsed();
+        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
+
         res
     }
 
@@ -354,7 +353,7 @@ impl Context {
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<MsgId> {
+    pub fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -362,7 +361,7 @@ impl Context {
         let strLikeInText = format!("%{}%", real_query);
         let strLikeBeg = format!("{}%", real_query);
 
-        let query = if 0 != chat_id {
+        let query = if !chat_id.is_unset() {
             concat!(
                 "SELECT m.id AS id, m.timestamp AS timestamp",
                 " FROM msgs m",
@@ -394,7 +393,7 @@ impl Context {
         self.sql
             .query_map(
                 query,
-                params![chat_id as i32, &strLikeInText, &strLikeBeg],
+                params![chat_id, &strLikeInText, &strLikeBeg],
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -445,15 +444,17 @@ impl Context {
                 return;
             }
 
-            // 1 = dc message, 2 = reply to dc message
-            if 0 != msg.is_dc_message {
-                job_add(
-                    self,
-                    Action::MoveMsg,
-                    msg.id.to_u32() as i32,
-                    Params::new(),
-                    0,
-                );
+            match msg.is_dc_message {
+                MessengerMessage::No => {}
+                MessengerMessage::Yes | MessengerMessage::Reply => {
+                    job_add(
+                        self,
+                        Action::MoveMsg,
+                        msg.id.to_u32() as i32,
+                        Params::new(),
+                        0,
+                    );
+                }
             }
         }
     }
@@ -483,14 +484,14 @@ impl Default for RunningState {
 }
 
 #[derive(Debug, Default)]
-pub struct BobStatus {
+pub(crate) struct BobStatus {
     pub expects: i32,
     pub status: i32,
     pub qr_scan: Option<Lot>,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum PerformJobsNeeded {
+pub(crate) enum PerformJobsNeeded {
     Not,
     AtOnce,
     AvoidDos,
@@ -507,7 +508,7 @@ pub struct SmtpState {
     pub idle: bool,
     pub suspended: bool,
     pub doing_jobs: bool,
-    pub perform_jobs_needed: PerformJobsNeeded,
+    pub(crate) perform_jobs_needed: PerformJobsNeeded,
     pub probe_network: bool,
 }
 
@@ -526,7 +527,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| 0), "FakeOs".into(), dbfile);
+        let res = Context::new(Box::new(|_, _| ()), "FakeOs".into(), dbfile);
         assert!(res.is_err());
     }
 
@@ -541,7 +542,7 @@ mod tests {
     fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
+        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -552,7 +553,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile);
+        let res = Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile);
         assert!(res.is_err());
     }
 
@@ -562,7 +563,7 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
+        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -572,7 +573,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 
@@ -581,7 +582,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 

src/dc_simplify.rs

@@ -1,273 +0,0 @@
-use crate::dehtml::*;
-
-#[derive(Copy, Clone)]
-pub struct Simplify {
-    pub is_forwarded: bool,
-}
-
-/// Return index of footer line in vector of message lines, or vector length if
-/// no footer is found.
-///
-/// Also return whether not-standard (rfc3676, §4.3) footer is found.
-fn find_message_footer(lines: &[&str]) -> (usize, bool) {
-    for (ix, &line) in lines.iter().enumerate() {
-        // quoted-printable may encode `-- ` to `-- =20` which is converted
-        // back to `--  `
-        match line {
-            "-- " | "--  " => return (ix, false),
-            "--" | "---" | "----" => return (ix, true),
-            _ => (),
-        }
-    }
-    (lines.len(), false)
-}
-
-impl Simplify {
-    pub fn new() -> Self {
-        Simplify {
-            is_forwarded: false,
-        }
-    }
-
-    /// Simplify and normalise text: Remove quotes, signatures, unnecessary
-    /// lineends etc.
-    /// The data returned from simplify() must be free()'d when no longer used.
-    pub fn simplify(&mut self, input: &str, is_html: bool, is_msgrmsg: bool) -> String {
-        let mut out = if is_html {
-            dehtml(input)
-        } else {
-            input.to_string()
-        };
-
-        out.retain(|c| c != '\r');
-        out = self.simplify_plain_text(&out, is_msgrmsg);
-        out.retain(|c| c != '\r');
-
-        out
-    }
-
-    /**
-     * Simplify Plain Text
-     */
-    #[allow(non_snake_case)]
-    fn simplify_plain_text(&mut self, buf_terminated: &str, is_msgrmsg: bool) -> String {
-        /* This function ...
-        ... removes all text after the line `-- ` (footer mark)
-        ... removes full quotes at the beginning and at the end of the text -
-            these are all lines starting with the character `>`
-        ... remove a non-empty line before the removed quote (contains sth. like "On 2.9.2016, Bjoern wrote:" in different formats and lanugages) */
-        /* split the given buffer into lines */
-        let lines: Vec<_> = buf_terminated.split('\n').collect();
-        let mut l_first: usize = 0;
-        let mut is_cut_at_begin = false;
-        let (mut l_last, mut is_cut_at_end) = find_message_footer(&lines);
-
-        if l_last > l_first + 2 {
-            let line0 = lines[l_first];
-            let line1 = lines[l_first + 1];
-            let line2 = lines[l_first + 2];
-            if line0 == "---------- Forwarded message ----------"
-                && line1.starts_with("From: ")
-                && line2.is_empty()
-            {
-                self.is_forwarded = true;
-                l_first += 3
-            }
-        }
-        for l in l_first..l_last {
-            let line = lines[l];
-            if line == "-----"
-                || line == "_____"
-                || line == "====="
-                || line == "*****"
-                || line == "~~~~~"
-            {
-                l_last = l;
-                is_cut_at_end = true;
-                /* done */
-                break;
-            }
-        }
-        if !is_msgrmsg {
-            let mut l_lastQuotedLine = None;
-            for l in (l_first..l_last).rev() {
-                let line = lines[l];
-                if is_plain_quote(line) {
-                    l_lastQuotedLine = Some(l)
-                } else if !is_empty_line(line) {
-                    break;
-                }
-            }
-            if let Some(last_quoted_line) = l_lastQuotedLine {
-                l_last = last_quoted_line;
-                is_cut_at_end = true;
-                if l_last > 1 && is_empty_line(lines[l_last - 1]) {
-                    l_last -= 1
-                }
-                if l_last > 1 {
-                    let line = lines[l_last - 1];
-                    if is_quoted_headline(line) {
-                        l_last -= 1
-                    }
-                }
-            }
-        }
-        if !is_msgrmsg {
-            let mut l_lastQuotedLine_0 = None;
-            let mut hasQuotedHeadline = 0;
-            for l in l_first..l_last {
-                let line = lines[l];
-                if is_plain_quote(line) {
-                    l_lastQuotedLine_0 = Some(l)
-                } else if !is_empty_line(line) {
-                    if is_quoted_headline(line)
-                        && 0 == hasQuotedHeadline
-                        && l_lastQuotedLine_0.is_none()
-                    {
-                        hasQuotedHeadline = 1i32
-                    } else {
-                        /* non-quoting line found */
-                        break;
-                    }
-                }
-            }
-            if let Some(last_quoted_line) = l_lastQuotedLine_0 {
-                l_first = last_quoted_line + 1;
-                is_cut_at_begin = true
-            }
-        }
-        /* re-create buffer from the remaining lines */
-        let mut ret = String::new();
-        if is_cut_at_begin {
-            ret += "[...]";
-        }
-        /* we write empty lines only in case and non-empty line follows */
-        let mut pending_linebreaks = 0;
-        let mut content_lines_added = 0;
-        for l in l_first..l_last {
-            let line = lines[l];
-            if is_empty_line(line) {
-                pending_linebreaks += 1
-            } else {
-                if 0 != content_lines_added {
-                    if pending_linebreaks > 2i32 {
-                        pending_linebreaks = 2i32
-                    }
-                    while 0 != pending_linebreaks {
-                        ret += "\n";
-                        pending_linebreaks -= 1
-                    }
-                }
-                // the incoming message might contain invalid UTF8
-                ret += line;
-                content_lines_added += 1;
-                pending_linebreaks = 1i32
-            }
-        }
-        if is_cut_at_end && (!is_cut_at_begin || 0 != content_lines_added) {
-            ret += " [...]";
-        }
-
-        ret
-    }
-}
-
-/**
- * Tools
- */
-fn is_empty_line(buf: &str) -> bool {
-    // XXX: can it be simplified to buf.chars().all(|c| c.is_whitespace())?
-    //
-    // Strictly speaking, it is not equivalent (^A is not whitespace, but less than ' '),
-    // but having control sequences in email body?!
-    //
-    // See discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
-    for c in buf.chars() {
-        if c > ' ' {
-            return false;
-        }
-    }
-
-    true
-}
-
-fn is_quoted_headline(buf: &str) -> bool {
-    /* This function may be called for the line _directly_ before a quote.
-    The function checks if the line contains sth. like "On 01.02.2016, xy@z wrote:" in various languages.
-    - Currently, we simply check if the last character is a ':'.
-    - Checking for the existence of an email address may fail (headlines may show the user's name instead of the address) */
-
-    buf.len() <= 80 && buf.ends_with(':')
-}
-
-fn is_plain_quote(buf: &str) -> bool {
-    buf.starts_with('>')
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use proptest::prelude::*;
-
-    proptest! {
-        #[test]
-        // proptest does not support [[:graphical:][:space:]] regex.
-        fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let output = Simplify::new().simplify_plain_text(&input, true);
-            assert!(output.split('\n').all(|s| s != "-- "));
-        }
-    }
-
-    #[test]
-    fn test_simplify_trim() {
-        let mut simplify = Simplify::new();
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(plain, "line1\nline2");
-    }
-
-    #[test]
-    fn test_simplify_parse_href() {
-        let mut simplify = Simplify::new();
-        let html = "<a href=url>text</a";
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(plain, "[text](url)");
-    }
-
-    #[test]
-    fn test_simplify_bold_text() {
-        let mut simplify = Simplify::new();
-        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(plain, "text *bold*<>");
-    }
-
-    #[test]
-    fn test_simplify_html_encoded() {
-        let mut simplify = Simplify::new();
-        let html =
-                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
-
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(
-            plain,
-            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
-        );
-    }
-
-    #[test]
-    fn test_simplify_utilities() {
-        assert!(is_empty_line(" \t"));
-        assert!(is_empty_line(""));
-        assert!(is_empty_line(" \r"));
-        assert!(!is_empty_line(" x"));
-        assert!(is_plain_quote("> hello world"));
-        assert!(is_plain_quote(">>"));
-        assert!(!is_plain_quote("Life is pain"));
-        assert!(!is_plain_quote(""));
-    }
-}

src/dc_strencode.rs

@@ -1,74 +0,0 @@
-use itertools::Itertools;
-
-/// Encode non-ascii-strings as `=?UTF-8?Q?Bj=c3=b6rn_Petersen?=`.
-/// Belongs to RFC 2047: https://tools.ietf.org/html/rfc2047
-///
-/// We do not fold at position 72; this would result in empty words as `=?utf-8?Q??=` which are correct,
-/// but cannot be displayed by some mail programs (eg. Android Stock Mail).
-/// however, this is not needed, as long as _one_ word is not longer than 72 characters.
-/// _if_ it is, the display may get weird.  This affects the subject only.
-/// the best solution wor all this would be if libetpan encodes the line as only libetpan knowns when a header line is full.
-///
-/// @param to_encode Null-terminated UTF-8-string to encode.
-/// @return Returns the encoded string which must be free()'d when no longed needed.
-///     On errors, NULL is returned.
-pub fn dc_encode_header_words(input: impl AsRef<str>) -> String {
-    let mut result = String::default();
-    for (_, group) in &input.as_ref().chars().group_by(|c| c.is_whitespace()) {
-        let word: String = group.collect();
-        result.push_str(&quote_word(&word.as_bytes()));
-    }
-
-    result
-}
-
-fn must_encode(byte: u8) -> bool {
-    static SPECIALS: &[u8] = b",:!\"#$@[\\]^`{|}~=?_";
-
-    SPECIALS.iter().any(|b| *b == byte)
-}
-
-fn quote_word(word: &[u8]) -> String {
-    let mut result = String::default();
-    let mut encoded = false;
-
-    for byte in word {
-        let byte = *byte;
-        if byte >= 128 || must_encode(byte) {
-            result.push_str(&format!("={:2X}", byte));
-            encoded = true;
-        } else if byte == b' ' {
-            result.push('_');
-            encoded = true;
-        } else {
-            result.push(byte as _);
-        }
-    }
-
-    if encoded {
-        result = format!("=?utf-8?Q?{}?=", &result);
-    }
-    result
-}
-
-/* ******************************************************************************
- * Encode/decode header words, RFC 2047
- ******************************************************************************/
-
-pub fn dc_needs_ext_header(to_check: impl AsRef<str>) -> bool {
-    let to_check = to_check.as_ref();
-
-    if to_check.is_empty() {
-        return false;
-    }
-
-    to_check.chars().any(|c| {
-        !(c.is_ascii_alphanumeric()
-            || c == '-'
-            || c == '_'
-            || c == '_'
-            || c == '.'
-            || c == '~'
-            || c == '%')
-    })
-}

src/dc_tools.rs

@@ -5,24 +5,24 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::time::SystemTime;
+use std::time::{Duration, SystemTime};
 use std::{fmt, fs};
 
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::Error;
+use crate::error::{bail, Error};
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
     0 != v && 0 == v & (v - 1)
 }
 
-/// Shortens a string to a specified length and adds "..." or "[...]" to the end of
-/// the shortened string.
-pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Cow<str> {
-    let ellipse = if do_unwrap { "..." } else { "[...]" };
+/// Shortens a string to a specified length and adds "[...]" to the
+/// end of the shortened string.
+pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
+    let ellipse = "[...]";
 
     let count = buf.chars().count();
     if approx_chars > 0 && count > approx_chars + ellipse.len() {
@@ -49,8 +49,8 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
 /// - harmonize together while being different enough
 /// (therefore, we cannot just use random rgb colors :)
 const COLORS: [u32; 16] = [
-    0xe56555, 0xf28c48, 0x8e85ee, 0x76c84d, 0x5bb6cc, 0x549cdd, 0xd25c99, 0xb37800, 0xf23030,
-    0x39b249, 0xbb243b, 0x964078, 0x66874f, 0x308ab9, 0x127ed0, 0xbe450c,
+    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
+    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
 ];
 
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
@@ -59,7 +59,7 @@ pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let bytes = str_lower.as_bytes();
     for (i, byte) in bytes.iter().enumerate() {
         checksum += (i + 1) * *byte as usize;
-        checksum %= 0xffffff;
+        checksum %= 0x00ff_ffff;
     }
     let color_index = checksum % COLORS.len();
 
@@ -75,6 +75,14 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String {
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
+pub fn duration_to_str(duration: Duration) -> String {
+    let secs = duration.as_secs();
+    let h = secs / 3600;
+    let m = (secs % 3600) / 60;
+    let s = (secs % 3600) % 60;
+    format!("{}h {}m {}s", h, m, s)
+}
+
 pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
@@ -182,22 +190,6 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
     String::from_utf8(wrapped_writer).unwrap()
 }
 
-pub(crate) fn dc_create_incoming_rfc724_mid(
-    message_timestamp: i64,
-    contact_id_from: u32,
-    contact_ids_to: &[u32],
-) -> Option<String> {
-    /* create a deterministic rfc724_mid from input such that
-    repeatedly calling it with the same input results in the same Message-id */
-
-    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
-    let result = format!(
-        "{}-{}-{}@stub",
-        message_timestamp, contact_id_from, largest_id_to
-    );
-    Some(result)
-}
-
 /// Function generates a Message-ID that can be used for a new outgoing message.
 /// - this function is called for all outgoing messages.
 /// - the message ID should be globally unique
@@ -217,8 +209,11 @@ pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str
 ///
 /// # Arguments
 ///
-/// * `mid` - A string that holds the message id
+/// * `mid` - A string that holds the message id.  Leading/Trailing <>
+/// characters are automatically stripped.
 pub(crate) fn dc_extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> {
+    let mid = mid.trim_start_matches('<').trim_end_matches('>');
+
     if mid.len() < 9 || !mid.starts_with("Gr.") {
         return None;
     }
@@ -235,58 +230,11 @@ pub(crate) fn dc_extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> {
     None
 }
 
-pub(crate) fn dc_ensure_no_slash_safe(path: &str) -> &str {
-    if path.ends_with('/') || path.ends_with('\\') {
-        return &path[..path.len() - 1];
-    }
-    path
-}
-
-// Function returns a sanitized basename that does not contain
-// win/linux path separators and also not any non-ascii chars
-fn get_safe_basename(filename: &str) -> String {
-    // return the (potentially mangled) basename of the input filename
-    // this might be a path that comes in from another operating system
-    let mut index: usize = 0;
-
-    if let Some(unix_index) = filename.rfind('/') {
-        index = unix_index + 1;
-    }
-    if let Some(win_index) = filename.rfind('\\') {
-        index = max(index, win_index + 1);
-    }
-    if index >= filename.len() {
-        "nobasename".to_string()
-    } else {
-        // we don't allow any non-ascii to be super-safe
-        filename[index..].replace(|c: char| !c.is_ascii() || c == ':', "-")
-    }
-}
-
-pub fn dc_derive_safe_stem_ext(filename: &str) -> (String, String) {
-    let basename = get_safe_basename(&filename);
-    let (mut stem, mut ext) = if let Some(index) = basename.rfind('.') {
-        (
-            basename[0..index].to_string(),
-            basename[index..].to_string(),
-        )
-    } else {
-        (basename, "".to_string())
-    };
-    // limit length of stem and ext
-    stem.truncate(32);
-    ext.truncate(32);
-    (stem, ext)
-}
-
 // the returned suffix is lower-case
-#[allow(non_snake_case)]
 pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
-    if let Some(p) = Path::new(path_filename.as_ref()).extension() {
-        Some(p.to_string_lossy().to_lowercase())
-    } else {
-        None
-    }
+    Path::new(path_filename.as_ref())
+        .extension()
+        .map(|p| p.to_string_lossy().to_lowercase())
 }
 
 /// Returns the `(width, height)` of the given image buffer.
@@ -403,11 +351,14 @@ pub(crate) fn dc_copy_file(
     }
 }
 
-pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
+pub(crate) fn dc_create_folder(
+    context: &Context,
+    path: impl AsRef<std::path::Path>,
+) -> Result<(), std::io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs) {
-            Ok(_) => true,
+            Ok(_) => Ok(()),
             Err(err) => {
                 warn!(
                     context,
@@ -415,11 +366,11 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
                     path.as_ref().display(),
                     err
                 );
-                false
+                Err(err)
             }
         }
     } else {
-        true
+        Ok(())
     }
 }
 
@@ -505,10 +456,27 @@ pub(crate) fn dc_get_next_backup_path(
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap()
+        .unwrap_or_default()
         .as_secs() as i64
 }
 
+/// An invalid email address was encountered
+#[derive(Debug, thiserror::Error)]
+#[error("Invalid email address: {message} ({addr})")]
+pub struct InvalidEmailError {
+    message: String,
+    addr: String,
+}
+
+impl InvalidEmailError {
+    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
+        InvalidEmailError {
+            message: msg.into(),
+            addr: addr.into(),
+        }
+    }
+}
+
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -525,14 +493,14 @@ pub(crate) fn time() -> i64 {
 /// assert_eq!(&email.domain, "example.com");
 /// assert_eq!(email.to_string(), "someone@example.com");
 /// ```
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Eq, Clone)]
 pub struct EmailAddress {
     pub local: String,
     pub domain: String,
 }
 
 impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, Error> {
+    pub fn new(input: &str) -> Result<Self, InvalidEmailError> {
         input.parse::<EmailAddress>()
     }
 }
@@ -544,31 +512,55 @@ impl fmt::Display for EmailAddress {
 }
 
 impl FromStr for EmailAddress {
-    type Err = Error;
+    type Err = InvalidEmailError;
 
     /// Performs a dead-simple parse of an email address.
-    fn from_str(input: &str) -> Result<EmailAddress, Error> {
-        ensure!(!input.is_empty(), "empty string is not valid");
+    fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
+        if input.is_empty() {
+            return Err(InvalidEmailError::new("empty string is not valid", input));
+        }
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
-        ensure!(parts.len() > 1, "missing '@' character");
-        let local = parts[1];
-        let domain = parts[0];
+        let err = |msg: &str| {
+            Err(InvalidEmailError {
+                message: msg.to_string(),
+                addr: input.to_string(),
+            })
+        };
+        match &parts[..] {
+            [domain, local] => {
+                if local.is_empty() {
+                    return err("empty string is not valid for local part");
+                }
+                if domain.len() <= 3 {
+                    return err("domain is too short");
+                }
+                let dot = domain.find('.');
+                match dot {
+                    None => {
+                        return err("invalid domain");
+                    }
+                    Some(dot_idx) => {
+                        if dot_idx >= domain.len() - 2 {
+                            return err("invalid domain");
+                        }
+                    }
+                }
+                Ok(EmailAddress {
+                    local: (*local).to_string(),
+                    domain: (*domain).to_string(),
+                })
+            }
+            _ => err("missing '@' character"),
+        }
+    }
+}
 
-        ensure!(
-            !local.is_empty(),
-            "empty string is not valid for local part"
-        );
-        ensure!(domain.len() > 3, "domain is too short");
-
-        let dot = domain.find('.');
-        ensure!(dot.is_some(), "invalid domain");
-        ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
-
-        Ok(EmailAddress {
-            local: local.to_string(),
-            domain: domain.to_string(),
-        })
+impl rusqlite::types::ToSql for EmailAddress {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Text(self.to_string());
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
     }
 }
 
@@ -595,54 +587,42 @@ mod tests {
     #[test]
     fn test_dc_truncate_1() {
         let s = "this is a little test string";
-        assert_eq!(dc_truncate(s, 16, false), "this is a [...]");
-        assert_eq!(dc_truncate(s, 16, true), "this is a ...");
+        assert_eq!(dc_truncate(s, 16), "this is a [...]");
     }
 
     #[test]
     fn test_dc_truncate_2() {
-        assert_eq!(dc_truncate("1234", 2, false), "1234");
-        assert_eq!(dc_truncate("1234", 2, true), "1234");
+        assert_eq!(dc_truncate("1234", 2), "1234");
     }
 
     #[test]
     fn test_dc_truncate_3() {
-        assert_eq!(dc_truncate("1234567", 1, false), "1[...]");
-        assert_eq!(dc_truncate("1234567", 1, true), "1...");
+        assert_eq!(dc_truncate("1234567", 1), "1[...]");
     }
 
     #[test]
     fn test_dc_truncate_4() {
-        assert_eq!(dc_truncate("123456", 4, false), "123456");
-        assert_eq!(dc_truncate("123456", 4, true), "123456");
+        assert_eq!(dc_truncate("123456", 4), "123456");
     }
 
     #[test]
     fn test_dc_truncate_edge() {
-        assert_eq!(dc_truncate("", 4, false), "");
-        assert_eq!(dc_truncate("", 4, true), "");
+        assert_eq!(dc_truncate("", 4), "");
 
-        assert_eq!(dc_truncate("\n  hello \n world", 4, false), "\n  [...]");
-        assert_eq!(dc_truncate("\n  hello \n world", 4, true), "\n  ...");
+        assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
 
+        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
         assert_eq!(
-            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1, false),
-            "𐠈[...]"
-        );
-        assert_eq!(
-            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0, false),
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
             "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
         );
 
         // 9 characters, so no truncation
-        assert_eq!(
-            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6, false),
-            "𑒀ὐ￠🜀\u{1e01b}A a🟠",
-        );
+        assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
 
         // 12 characters, truncation
         assert_eq!(
-            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6, false),
+            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6),
             "𑒀ὐ￠🜀\u{1e01b}A[...]",
         );
     }
@@ -673,7 +653,6 @@ mod tests {
         );
     }
 
-    #[test]
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid
@@ -695,6 +674,16 @@ mod tests {
         let mid = "Gr.1234567890123456.morerandom@domain.de";
         let grpid = dc_extract_grpid_from_rfc724_mid(mid);
         assert_eq!(grpid, Some("1234567890123456"));
+
+        // Should return extracted grpid for grpid with length of 11
+        let mid = "<Gr.12345678901.morerandom@domain.de>";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, Some("12345678901"));
+
+        // Should return extracted grpid for grpid with length of 11
+        let mid = "<Gr.1234567890123456.morerandom@domain.de>";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, Some("1234567890123456"));
     }
 
     #[test]
@@ -718,29 +707,29 @@ mod tests {
 
     #[test]
     fn test_emailaddress_parse() {
-        assert_eq!(EmailAddress::new("").is_ok(), false);
+        assert_eq!("".parse::<EmailAddress>().is_ok(), false);
         assert_eq!(
-            EmailAddress::new("user@domain.tld").unwrap(),
+            "user@domain.tld".parse::<EmailAddress>().unwrap(),
             EmailAddress {
                 local: "user".into(),
                 domain: "domain.tld".into(),
             }
         );
-        assert_eq!(EmailAddress::new("uuu").is_ok(), false);
-        assert_eq!(EmailAddress::new("dd.tt").is_ok(), false);
-        assert_eq!(EmailAddress::new("tt.dd@uu").is_ok(), false);
-        assert_eq!(EmailAddress::new("u@d").is_ok(), false);
-        assert_eq!(EmailAddress::new("u@d.").is_ok(), false);
-        assert_eq!(EmailAddress::new("u@d.t").is_ok(), false);
+        assert_eq!("uuu".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("dd.tt".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("tt.dd@uu".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.t".parse::<EmailAddress>().is_ok(), false);
         assert_eq!(
-            EmailAddress::new("u@d.tt").unwrap(),
+            "u@d.tt".parse::<EmailAddress>().unwrap(),
             EmailAddress {
                 local: "u".into(),
                 domain: "d.tt".into(),
             }
         );
-        assert_eq!(EmailAddress::new("u@.tt").is_ok(), false);
-        assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
+        assert_eq!("u@tt".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
     use proptest::prelude::*;
@@ -749,11 +738,10 @@ mod tests {
         #[test]
         fn test_dc_truncate(
             buf: String,
-            approx_chars in 0..10000usize,
-            do_unwrap: bool,
+            approx_chars in 0..10000usize
         ) {
-            let res = dc_truncate(&buf, approx_chars, do_unwrap);
-            let el_len = if do_unwrap { 3 } else { 5 };
+            let res = dc_truncate(&buf, approx_chars);
+            let el_len = 5;
             let l = res.chars().count();
             if approx_chars > 0 {
                 assert!(
@@ -767,36 +755,11 @@ mod tests {
 
             if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
                 let l = res.len();
-                if do_unwrap {
-                    assert_eq!(&res[l-3..l], "...", "missing ellipsis in {}", &res);
-                } else {
-                    assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
-                }
+                assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
             }
         }
     }
 
-    #[test]
-    fn test_dc_create_incoming_rfc724_mid() {
-        let res = dc_create_incoming_rfc724_mid(123, 45, &vec![6, 7]);
-        assert_eq!(res, Some("123-45-7@stub".into()));
-        let res = dc_create_incoming_rfc724_mid(123, 45, &vec![]);
-        assert_eq!(res, Some("123-45-0@stub".into()));
-    }
-
-    #[test]
-    fn test_file_get_safe_basename() {
-        assert_eq!(get_safe_basename("12312/hello"), "hello");
-        assert_eq!(get_safe_basename("12312\\hello"), "hello");
-        assert_eq!(get_safe_basename("//12312\\hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12\\hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12/\\\\hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12//hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12//"), "nobasename");
-        assert_eq!(get_safe_basename("//123:12/"), "nobasename");
-        assert!(get_safe_basename("123\x012.hello").ends_with(".hello"));
-    }
-
     #[test]
     fn test_file_handling() {
         let t = dummy_context();
@@ -845,7 +808,7 @@ mod tests {
 
         assert!(dc_delete_file(context, "$BLOBDIR/foobar"));
         assert!(dc_delete_file(context, "$BLOBDIR/dada"));
-        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder"));
+        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder").is_ok());
         assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
 
@@ -859,15 +822,15 @@ mod tests {
     #[test]
     fn test_listflags_has() {
         let listflags: u32 = 0x1101;
-        assert!(listflags_has(listflags, 0x1) == true);
-        assert!(listflags_has(listflags, 0x10) == false);
-        assert!(listflags_has(listflags, 0x100) == true);
-        assert!(listflags_has(listflags, 0x1000) == true);
+        assert!(listflags_has(listflags, 0x1));
+        assert!(!listflags_has(listflags, 0x10));
+        assert!(listflags_has(listflags, 0x100));
+        assert!(listflags_has(listflags, 0x1000));
         let listflags: u32 = (DC_GCL_ADD_SELF | DC_GCL_VERIFIED_ONLY).try_into().unwrap();
-        assert!(listflags_has(listflags, DC_GCL_VERIFIED_ONLY) == true);
-        assert!(listflags_has(listflags, DC_GCL_ADD_SELF) == true);
+        assert!(listflags_has(listflags, DC_GCL_VERIFIED_ONLY));
+        assert!(listflags_has(listflags, DC_GCL_ADD_SELF));
         let listflags: u32 = DC_GCL_VERIFIED_ONLY.try_into().unwrap();
-        assert!(listflags_has(listflags, DC_GCL_ADD_SELF) == false);
+        assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
     }
 
     #[test]
@@ -899,4 +862,37 @@ mod tests {
         let next = dc_smeared_time(&t.ctx);
         assert!((start + count - 1) < next);
     }
+
+    #[test]
+    fn test_duration_to_str() {
+        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
+        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
+        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
+        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
+        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
+        assert_eq!(
+            duration_to_str(Duration::from_secs(59 * 60 + 59)),
+            "0h 59m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(59 * 60 + 60)),
+            "1h 0m 0s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
+            "2h 59m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
+            "3h 0m 0s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
+            "3h 0m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
+            "3h 1m 0s"
+        );
+    }
 }

src/dehtml.rs

@@ -3,7 +3,6 @@
 //! A module to remove HTML tags from the email text
 
 use lazy_static::lazy_static;
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 lazy_static! {
@@ -35,6 +34,7 @@ pub fn dehtml(buf: &str) -> String {
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
+    reader.check_end_names(false);
 
     let mut buf = Vec::new();
 
@@ -139,13 +139,12 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
-            if let Some(href) = event.html_attributes().find(|attr| {
-                attr.as_ref()
-                    .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "href")
-                    .unwrap_or_default()
-            }) {
+            if let Some(href) = event
+                .html_attributes()
+                .filter_map(|attr| attr.ok())
+                .find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "href")
+            {
                 let href = href
-                    .unwrap()
                     .unescape_and_decode_value(reader)
                     .unwrap_or_default()
                     .to_lowercase();
@@ -189,4 +188,60 @@ mod tests {
             assert_eq!(dehtml(input), output);
         }
     }
+
+    #[test]
+    fn test_dehtml_parse_br() {
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
+        let plain = dehtml(html);
+
+        assert_eq!(plain, "line1\n\r\r\rline2");
+    }
+
+    #[test]
+    fn test_dehtml_parse_href() {
+        let html = "<a href=url>text</a";
+        let plain = dehtml(html);
+
+        assert_eq!(plain, "[text](url)");
+    }
+
+    #[test]
+    fn test_dehtml_bold_text() {
+        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
+        let plain = dehtml(html);
+
+        assert_eq!(plain, "text *bold*<>");
+    }
+
+    #[test]
+    fn test_dehtml_html_encoded() {
+        let html =
+                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
+
+        let plain = dehtml(html);
+
+        assert_eq!(
+            plain,
+            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
+        );
+    }
+
+    #[test]
+    fn test_unclosed_tags() {
+        let input = r##"
+        <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'
+        'http://www.w3.org/TR/html4/loose.dtd'>
+        <html>
+        <head>
+        <title>Hi</title>
+        <meta http-equiv='Content-Type' content='text/html; charset=iso-8859-1'>						
+        </head>
+        <body>
+        lots of text
+        </body>
+        </html>
+        "##;
+        let txt = dehtml(input);
+        assert_eq!(txt.trim(), "lots of text");
+    }
 }

src/e2ee.rs

@@ -2,25 +2,25 @@
 
 use std::collections::HashSet;
 
-use mailparse::MailHeaderMap;
+use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
 use crate::error::*;
-use crate::key::*;
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
 use crate::securejoin::handle_degrade_event;
-use crate::wrapmime;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
     pub prefer_encrypt: EncryptPreference,
     pub addr: String,
-    pub public_key: Key,
+    pub public_key: SignedPublicKey,
 }
 
 impl EncryptHelper {
@@ -35,7 +35,7 @@ impl EncryptHelper {
             Some(addr) => addr,
         };
 
-        let public_key = load_or_generate_self_public_key(context, &addr)?;
+        let public_key = SignedPublicKey::load_self(context)?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -98,16 +98,14 @@ impl EncryptHelper {
             .iter()
             .filter_map(|(state, addr)| state.as_ref().map(|s| (s, addr)))
         {
-            info!(context, "adding for {}: {:?}", addr, peerstate);
             let key = peerstate.peek_key(min_verified).ok_or_else(|| {
                 format_err!("proper enc-key for {} missing, cannot encrypt", addr)
             })?;
             keyring.add_ref(key);
         }
-
-        keyring.add_ref(&self.public_key);
-        let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql)
-            .ok_or_else(|| format_err!("missing own private key"))?;
+        let public_key = Key::from(self.public_key.clone());
+        keyring.add_ref(&public_key);
+        let sign_key = Key::from(SignedSecretKey::load_self(context)?);
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
@@ -119,14 +117,12 @@ impl EncryptHelper {
 
 pub fn try_decrypt(
     context: &Context,
-    mail: &mailparse::ParsedMail<'_>,
+    mail: &ParsedMail<'_>,
     message_time: i64,
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
-    info!(context, "trying to decrypt");
-
     let from = mail
         .headers
-        .get_first_value("From")?
+        .get_header_value(HeaderDef::From_)
         .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
@@ -189,56 +185,36 @@ pub fn try_decrypt(
     Ok((out_mail, signatures))
 }
 
-/// Load public key from database or generate a new one.
-///
-/// This will load a public key from the database, generating and
-/// storing a new one when one doesn't exist yet.  Care is taken to
-/// only generate one key per context even when multiple threads call
-/// this function concurrently.
-fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str>) -> Result<Key> {
-    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
-        return Ok(key);
-    }
-    let _guard = context.generating_key_mutex.lock().unwrap();
-
-    // Check again in case the key was generated while we were waiting for the lock.
-    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
-        return Ok(key);
-    }
-
-    let start = std::time::Instant::now();
-    info!(
-        context,
-        "Generating keypair with {} bits, e={} ...", 2048, 65537,
+/// Returns a reference to the encrypted payload and validates the autocrypt structure.
+fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
+    ensure!(
+        mail.ctype.mimetype == "multipart/encrypted",
+        "Not a multipart/encrypted message: {}",
+        mail.ctype.mimetype
     );
-    match pgp::create_keypair(&self_addr) {
-        Some((public_key, private_key)) => {
-            match dc_key_save_self_keypair(
-                context,
-                &public_key,
-                &private_key,
-                &self_addr,
-                true,
-                &context.sql,
-            ) {
-                true => {
-                    info!(
-                        context,
-                        "Keypair generated in {:.3}s.",
-                        start.elapsed().as_secs()
-                    );
-                    Ok(public_key)
-                }
-                false => Err(format_err!("Failed to save keypair")),
-            }
-        }
-        None => Err(format_err!("Failed to generate keypair")),
-    }
+    ensure!(
+        mail.subparts.len() == 2,
+        "Invalid Autocrypt Level 1 Mime Parts"
+    );
+
+    ensure!(
+        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
+        "Invalid Autocrypt Level 1 version part: {:?}",
+        mail.subparts[0].ctype,
+    );
+
+    ensure!(
+        mail.subparts[1].ctype.mimetype == "application/octet-stream",
+        "Invalid Autocrypt Level 1 encrypted part: {:?}",
+        mail.subparts[1].ctype
+    );
+
+    Ok(&mail.subparts[1])
 }
 
 fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
-    mail: &mailparse::ParsedMail<'a>,
+    mail: &ParsedMail<'a>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
@@ -250,14 +226,14 @@ fn decrypt_if_autocrypt_message<'a>(
     //
     // Errors are returned for failures related to decryption of AC-messages.
 
-    let encrypted_data_part = match wrapmime::get_autocrypt_mime(mail) {
-        Err(err) => {
-            // not a proper autocrypt message, abort and ignore
-            warn!(context, "Invalid autocrypt message: {:?}", err);
+    let encrypted_data_part = match get_autocrypt_mime(mail) {
+        Err(_) => {
+            // not an autocrypt mime message, abort and ignore
             return Ok(None);
         }
         Ok(res) => res,
     };
+    info!(context, "Detected Autocrypt-mime message");
 
     decrypt_part(
         context,
@@ -270,13 +246,12 @@ fn decrypt_if_autocrypt_message<'a>(
 
 /// Returns Ok(None) if nothing encrypted was found.
 fn decrypt_part(
-    context: &Context,
-    mail: &mailparse::ParsedMail<'_>,
+    _context: &Context,
+    mail: &ParsedMail<'_>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
 ) -> Result<Option<Vec<u8>>> {
-    info!(context, "decrypting part");
     let data = mail.get_body_raw()?;
 
     if has_decrypted_pgp_armor(&data) {
@@ -318,7 +293,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
 /// However, Delta Chat itself has no problem with encrypted multipart/report
 /// parts and MUAs should be encouraged to encrpyt multipart/reports as well so
 /// that we could use the normal Autocrypt processing.
-fn contains_report(mail: &mailparse::ParsedMail<'_>) -> bool {
+fn contains_report(mail: &ParsedMail<'_>) -> bool {
     mail.ctype.mimetype == "multipart/report"
 }
 
@@ -331,6 +306,7 @@ fn contains_report(mail: &mailparse::ParsedMail<'_>) -> bool {
 ///
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
+// TODO, remove this once deltachat::key::Key no longer exists.
 pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
         format_err!(concat!(
@@ -338,7 +314,7 @@ pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
             "cannot ensure secret key if not configured."
         ))
     })?;
-    load_or_generate_self_public_key(context, &self_addr)?;
+    SignedPublicKey::load_self(context)?;
     Ok(self_addr)
 }
 
@@ -389,49 +365,6 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
-    mod load_or_generate_self_public_key {
-        use super::*;
-
-        #[test]
-        fn test_existing() {
-            let t = dummy_context();
-            let addr = configure_alice_keypair(&t.ctx);
-            let key = load_or_generate_self_public_key(&t.ctx, addr);
-            assert!(key.is_ok());
-        }
-
-        #[test]
-        #[ignore] // generating keys is expensive
-        fn test_generate() {
-            let t = dummy_context();
-            let addr = "alice@example.org";
-            let key0 = load_or_generate_self_public_key(&t.ctx, addr);
-            assert!(key0.is_ok());
-            let key1 = load_or_generate_self_public_key(&t.ctx, addr);
-            assert!(key1.is_ok());
-            assert_eq!(key0.unwrap(), key1.unwrap());
-        }
-
-        #[test]
-        #[ignore]
-        fn test_generate_concurrent() {
-            use std::sync::Arc;
-            use std::thread;
-
-            let t = dummy_context();
-            let ctx = Arc::new(t.ctx);
-            let ctx0 = Arc::clone(&ctx);
-            let thr0 =
-                thread::spawn(move || load_or_generate_self_public_key(&ctx0, "alice@example.org"));
-            let ctx1 = Arc::clone(&ctx);
-            let thr1 =
-                thread::spawn(move || load_or_generate_self_public_key(&ctx1, "alice@example.org"));
-            let res0 = thr0.join().unwrap();
-            let res1 = thr1.join().unwrap();
-            assert_eq!(res0.unwrap(), res1.unwrap());
-        }
-    }
-
     #[test]
     fn test_has_decrypted_pgp_armor() {
         let data = b" -----BEGIN PGP MESSAGE-----";

src/error.rs

@@ -1,153 +1,13 @@
-use failure::Fail;
-use lettre_email::mime;
+//! # Error handling
 
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "{:?}", _0)]
-    Failure(failure::Error),
-    #[fail(display = "SQL error: {:?}", _0)]
-    SqlError(#[cause] crate::sql::Error),
-    #[fail(display = "{:?}", _0)]
-    Io(std::io::Error),
-    #[fail(display = "{:?}", _0)]
-    Message(String),
-    #[fail(display = "{:?}", _0)]
-    Image(image_meta::ImageError),
-    #[fail(display = "{:?}", _0)]
-    Utf8(std::str::Utf8Error),
-    #[fail(display = "PGP: {:?}", _0)]
-    Pgp(pgp::errors::Error),
-    #[fail(display = "Base64Decode: {:?}", _0)]
-    Base64Decode(base64::DecodeError),
-    #[fail(display = "{:?}", _0)]
-    FromUtf8(std::string::FromUtf8Error),
-    #[fail(display = "{}", _0)]
-    BlobError(#[cause] crate::blob::BlobError),
-    #[fail(display = "Invalid Message ID.")]
-    InvalidMsgId,
-    #[fail(display = "Watch folder not found {:?}", _0)]
-    WatchFolderNotFound(String),
-    #[fail(display = "Inalid Email: {:?}", _0)]
-    MailParseError(#[cause] mailparse::MailParseError),
-    #[fail(display = "Building invalid Email: {:?}", _0)]
-    LettreError(#[cause] lettre_email::error::Error),
-    #[fail(display = "FromStr error: {:?}", _0)]
-    FromStr(#[cause] mime::FromStrError),
-}
+pub use anyhow::{bail, ensure, format_err, Error, Result};
 
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<crate::sql::Error> for Error {
-    fn from(err: crate::sql::Error) -> Error {
-        Error::SqlError(err)
-    }
-}
-
-impl From<base64::DecodeError> for Error {
-    fn from(err: base64::DecodeError) -> Error {
-        Error::Base64Decode(err)
-    }
-}
-
-impl From<failure::Error> for Error {
-    fn from(err: failure::Error) -> Error {
-        Error::Failure(err)
-    }
-}
-
-impl From<std::io::Error> for Error {
-    fn from(err: std::io::Error) -> Error {
-        Error::Io(err)
-    }
-}
-
-impl From<std::str::Utf8Error> for Error {
-    fn from(err: std::str::Utf8Error) -> Error {
-        Error::Utf8(err)
-    }
-}
-
-impl From<image_meta::ImageError> for Error {
-    fn from(err: image_meta::ImageError) -> Error {
-        Error::Image(err)
-    }
-}
-
-impl From<pgp::errors::Error> for Error {
-    fn from(err: pgp::errors::Error) -> Error {
-        Error::Pgp(err)
-    }
-}
-
-impl From<std::string::FromUtf8Error> for Error {
-    fn from(err: std::string::FromUtf8Error) -> Error {
-        Error::FromUtf8(err)
-    }
-}
-
-impl From<crate::blob::BlobError> for Error {
-    fn from(err: crate::blob::BlobError) -> Error {
-        Error::BlobError(err)
-    }
-}
-
-impl From<crate::message::InvalidMsgId> for Error {
-    fn from(_err: crate::message::InvalidMsgId) -> Error {
-        Error::InvalidMsgId
-    }
-}
-
-impl From<mailparse::MailParseError> for Error {
-    fn from(err: mailparse::MailParseError) -> Error {
-        Error::MailParseError(err)
-    }
-}
-
-impl From<lettre_email::error::Error> for Error {
-    fn from(err: lettre_email::error::Error) -> Error {
-        Error::LettreError(err)
-    }
-}
-
-impl From<mime::FromStrError> for Error {
-    fn from(err: mime::FromStrError) -> Error {
-        Error::FromStr(err)
-    }
-}
-
-#[macro_export]
-macro_rules! bail {
-    ($e:expr) => {
-        return Err($crate::error::Error::Message($e.to_string()));
-    };
-    ($fmt:expr, $($arg:tt)+) => {
-        return Err($crate::error::Error::Message(format!($fmt, $($arg)+)));
-    };
-}
-
-#[macro_export]
-macro_rules! format_err {
-    ($e:expr) => {
-        $crate::error::Error::Message($e.to_string());
-    };
-    ($fmt:expr, $($arg:tt)+) => {
-        $crate::error::Error::Message(format!($fmt, $($arg)+));
-    };
-}
-
-#[macro_export(local_inner_macros)]
-macro_rules! ensure {
-    ($cond:expr, $e:expr) => {
-        if !($cond) {
-            bail!($e);
-        }
-    };
-    ($cond:expr, $fmt:expr, $($arg:tt)+) => {
-        if !($cond) {
-            bail!($fmt, $($arg)+);
-        }
-    };
-}
+// #[fail(display = "Invalid Message ID.")]
+// InvalidMsgId,
+// #[fail(display = "Watch folder not found {:?}", _0)]
+// WatchFolderNotFound(String),
+// #[fail(display = "Not Configured")]
+// NotConfigured,
 
 #[macro_export]
 macro_rules! ensure_eq {

src/events.rs

@@ -1,7 +1,10 @@
+//! # Events specification
+
 use std::path::PathBuf;
 
 use strum::EnumProperty;
 
+use crate::chat::ChatId;
 use crate::message::MsgId;
 
 impl Event {
@@ -19,56 +22,38 @@ pub enum Event {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
-    ///
-    /// @return 0
     #[strum(props(id = "100"))]
     Info(String),
 
     /// Emitted when SMTP connection is established and login was successful.
-    ///
-    /// @return 0
     #[strum(props(id = "101"))]
     SmtpConnected(String),
 
     /// Emitted when IMAP connection is established and login was successful.
-    ///
-    /// @return 0
     #[strum(props(id = "102"))]
     ImapConnected(String),
 
     /// Emitted when a message was successfully sent to the SMTP server.
-    ///
-    /// @return 0
     #[strum(props(id = "103"))]
     SmtpMessageSent(String),
 
     /// Emitted when an IMAP message has been marked as deleted
-    ///
-    /// @return 0
     #[strum(props(id = "104"))]
     ImapMessageDeleted(String),
 
     /// Emitted when an IMAP message has been moved
-    ///
-    /// @return 0
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
     /// Emitted when an IMAP folder was emptied
-    ///
-    /// @return 0
     #[strum(props(id = "106"))]
     ImapFolderEmptied(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
-    ///
-    /// @return 0
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
-    ///
-    /// @return 0
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -76,8 +61,6 @@ pub enum Event {
     /// Passed to the callback given to dc_context_new().
     ///
     /// This event should not be reported to the end-user using a popup or something like that.
-    ///
-    /// @return 0
     #[strum(props(id = "300"))]
     Warning(String),
 
@@ -90,10 +73,8 @@ pub enum Event {
     /// However, for ongoing processes (eg. configure())
     /// or for functions that are expected to fail (eg. dc_continue_key_transfer())
     /// it might be better to delay showing these events until the function has really
-    /// failed (returned false). It should be sufficient to report only the _last_ error
+    /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
-    ///
-    /// @return
     #[strum(props(id = "400"))]
     Error(String),
 
@@ -110,8 +91,6 @@ pub enum Event {
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
     /// instead of the string from data2.
-    ///
-    /// @return 0
     #[strum(props(id = "401"))]
     ErrorNetwork(String),
 
@@ -120,8 +99,6 @@ pub enum Event {
     /// dc_set_chat_name(), dc_set_chat_profile_image(),
     /// dc_add_contact_to_chat(), dc_remove_contact_from_chat(),
     /// dc_send_text_msg() or another sending function.
-    ///
-    /// @return 0
     #[strum(props(id = "410"))]
     ErrorSelfNotInGroup(String),
 
@@ -130,54 +107,41 @@ pub enum Event {
     /// - Messages sent, received or removed
     /// - Chats created, deleted or archived
     /// - A draft has been set
-    ///
-    /// @return 0
     #[strum(props(id = "2000"))]
-    MsgsChanged { chat_id: u32, msg_id: MsgId },
+    MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
     /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
-    ///
-    /// @return 0
     #[strum(props(id = "2005"))]
-    IncomingMsg { chat_id: u32, msg_id: MsgId },
+    IncomingMsg { chat_id: ChatId, msg_id: MsgId },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
-    ///
-    /// @return 0
     #[strum(props(id = "2010"))]
-    MsgDelivered { chat_id: u32, msg_id: MsgId },
+    MsgDelivered { chat_id: ChatId, msg_id: MsgId },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see dc_msg_get_state().
-    ///
-    /// @return 0
     #[strum(props(id = "2012"))]
-    MsgFailed { chat_id: u32, msg_id: MsgId },
+    MsgFailed { chat_id: ChatId, msg_id: MsgId },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
-    ///
-    /// @return 0
     #[strum(props(id = "2015"))]
-    MsgRead { chat_id: u32, msg_id: MsgId },
+    MsgRead { chat_id: ChatId, msg_id: MsgId },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
-    ///
-    /// @return 0
     #[strum(props(id = "2020"))]
-    ChatModified(u32),
+    ChatModified(ChatId),
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
-    /// @return 0
     #[strum(props(id = "2030"))]
     ContactsChanged(Option<u32>),
 
@@ -186,14 +150,12 @@ pub enum Event {
     /// @param data1 (u32) contact_id of the contact for which the location has changed.
     ///     If the locations of several contacts have been changed,
     ///     eg. after calling dc_delete_all_locations(), this parameter is set to `None`.
-    /// @return 0
     #[strum(props(id = "2035"))]
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
-    /// @return 0
     #[strum(props(id = "2041"))]
     ConfigureProgress(usize),
 
@@ -201,7 +163,6 @@ pub enum Event {
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
-    /// @return 0
     #[strum(props(id = "2051"))]
     ImexProgress(usize),
 
@@ -212,7 +173,6 @@ pub enum Event {
     /// services.
     ///
     /// @param data2 0
-    /// @return 0
     #[strum(props(id = "2052"))]
     ImexFileWritten(PathBuf),
 
@@ -228,7 +188,6 @@ pub enum Event {
     ///     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
-    /// @return 0
     #[strum(props(id = "2060"))]
     SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
@@ -240,14 +199,6 @@ pub enum Event {
     /// @param data2 (int) Progress as:
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
-    /// @return 0
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
-
-    /// This event is sent out to the inviter when a joiner successfully joined a group.
-    /// @param data1 (int) chat_id
-    /// @param data2 (int) contact_id
-    /// @return 0
-    #[strum(props(id = "2062"))]
-    SecurejoinMemberAdded { chat_id: u32, contact_id: u32 },
 }

src/headerdef.rs

@@ -0,0 +1,91 @@
+use crate::strum::AsStaticRef;
+use mailparse::{MailHeader, MailHeaderMap};
+
+#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
+#[strum(serialize_all = "kebab_case")]
+#[allow(dead_code)]
+pub enum HeaderDef {
+    MessageId,
+    Subject,
+    Date,
+    From_,
+    To,
+    Cc,
+    Disposition,
+    OriginalMessageId,
+
+    /// Delta Chat extension for message IDs in combined MDNs
+    AdditionalMessageIds,
+
+    ListId,
+    References,
+    InReplyTo,
+    Precedence,
+    ChatVersion,
+    ChatGroupId,
+    ChatGroupName,
+    ChatGroupNameChanged,
+    ChatVerified,
+    ChatGroupAvatar,
+    ChatUserAvatar,
+    ChatVoiceMessage,
+    ChatGroupMemberRemoved,
+    ChatGroupMemberAdded,
+    ChatContent,
+    ChatDuration,
+    ChatDispositionNotificationTo,
+    Autocrypt,
+    AutocryptSetupMessage,
+    SecureJoin,
+    SecureJoinGroup,
+    SecureJoinFingerprint,
+    SecureJoinInvitenumber,
+    SecureJoinAuth,
+    _TestHeader,
+}
+
+impl HeaderDef {
+    /// Returns the corresponding Event id.
+    pub fn get_headername(&self) -> &'static str {
+        self.as_static()
+    }
+}
+
+pub trait HeaderDefMap {
+    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
+}
+
+impl HeaderDefMap for [MailHeader<'_>] {
+    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
+        self.get_first_value(headerdef.get_headername())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// Test that kebab_case serialization works as expected
+    fn kebab_test() {
+        assert_eq!(HeaderDef::From_.get_headername(), "from");
+
+        assert_eq!(HeaderDef::_TestHeader.get_headername(), "test-header");
+    }
+
+    #[test]
+    /// Test that headers are parsed case-insensitively
+    fn test_get_header_value_case() {
+        let (headers, _) =
+            mailparse::parse_headers(b"fRoM: Bob\naUtoCryPt-SeTup-MessAge: v99").unwrap();
+        assert_eq!(
+            headers.get_header_value(HeaderDef::AutocryptSetupMessage),
+            Some("v99".to_string())
+        );
+        assert_eq!(
+            headers.get_header_value(HeaderDef::From_),
+            Some("Bob".to_string())
+        );
+        assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None);
+    }
+}

src/imap/client.rs

@@ -0,0 +1,104 @@
+use async_imap::{
+    error::{Error as ImapError, Result as ImapResult},
+    Client as ImapClient,
+};
+use async_native_tls::TlsStream;
+use async_std::net::{self, TcpStream};
+
+use super::session::Session;
+use crate::login_param::{dc_build_tls, CertificateChecks};
+
+#[derive(Debug)]
+pub(crate) enum Client {
+    Secure(ImapClient<TlsStream<TcpStream>>),
+    Insecure(ImapClient<TcpStream>),
+}
+
+impl Client {
+    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
+        addr: A,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+        let tls = dc_build_tls(certificate_checks);
+        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
+        let mut client = ImapClient::new(tls_stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Secure(client))
+    }
+
+    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+
+        let mut client = ImapClient::new(stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Insecure(client))
+    }
+
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
+        match self {
+            Client::Insecure(client) => {
+                let tls = dc_build_tls(certificate_checks);
+                let client_sec = client.secure(domain, tls).await?;
+
+                Ok(Client::Secure(client_sec))
+            }
+            // Nothing to do
+            Client::Secure(_) => Ok(self),
+        }
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+}

src/imap/idle.rs

@@ -0,0 +1,287 @@
+use super::Imap;
+
+use async_imap::extensions::idle::{Handle as ImapIdleHandle, IdleResponse};
+use async_native_tls::TlsStream;
+use async_std::net::TcpStream;
+use async_std::prelude::*;
+use async_std::task;
+use std::sync::atomic::Ordering;
+use std::time::{Duration, SystemTime};
+
+use crate::context::Context;
+
+use super::select_folder;
+use super::session::Session;
+
+type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("IMAP IDLE protocol failed to init/complete")]
+    IdleProtocolFailed(#[from] async_imap::error::Error),
+
+    #[error("IMAP IDLE protocol timed out")]
+    IdleTimeout(#[from] async_std::future::TimeoutError),
+
+    #[error("IMAP server does not have IDLE capability")]
+    IdleAbilityMissing,
+
+    #[error("IMAP select folder error")]
+    SelectFolderError(#[from] select_folder::Error),
+
+    #[error("Setup handle error")]
+    SetupHandleError(#[from] super::Error),
+}
+
+#[derive(Debug)]
+pub(crate) enum IdleHandle {
+    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
+    Insecure(ImapIdleHandle<TcpStream>),
+}
+
+impl Session {
+    pub fn idle(self) -> IdleHandle {
+        match self {
+            Session::Secure(i) => {
+                let h = i.idle();
+                IdleHandle::Secure(h)
+            }
+            Session::Insecure(i) => {
+                let h = i.idle();
+                IdleHandle::Insecure(h)
+            }
+        }
+    }
+}
+
+impl Imap {
+    pub fn can_idle(&self) -> bool {
+        task::block_on(async move { self.config.read().await.can_idle })
+    }
+
+    pub fn idle(&self, context: &Context, watch_folder: Option<String>) -> Result<()> {
+        task::block_on(async move {
+            if !self.can_idle() {
+                return Err(Error::IdleAbilityMissing);
+            }
+
+            self.setup_handle_if_needed(context).await?;
+
+            self.select_folder(context, watch_folder.clone()).await?;
+
+            let session = self.session.lock().await.take();
+            let timeout = Duration::from_secs(23 * 60);
+            if let Some(session) = session {
+                match session.idle() {
+                    // BEWARE: If you change the Secure branch you
+                    // typically also need to change the Insecure branch.
+                    IdleHandle::Secure(mut handle) => {
+                        handle.init().await?;
+
+                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
+                        *self.interrupt.lock().await = Some(interrupt);
+
+                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+                            // interrupt_idle has happened before we
+                            // provided self.interrupt
+                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
+                            std::mem::drop(idle_wait);
+                            info!(context, "Idle wait was skipped");
+                        } else {
+                            info!(context, "Idle entering wait-on-remote state");
+                            match idle_wait.await {
+                                Ok(IdleResponse::NewData(_)) => {
+                                    info!(context, "Idle has NewData");
+                                }
+                                // TODO: idle_wait does not distinguish manual interrupts
+                                // from Timeouts if we would know it's a Timeout we could bail
+                                // directly and reconnect .
+                                Ok(IdleResponse::Timeout) => {
+                                    info!(context, "Idle-wait timeout or interruption");
+                                }
+                                Ok(IdleResponse::ManualInterrupt) => {
+                                    info!(context, "Idle wait was interrupted");
+                                }
+                                Err(err) => {
+                                    warn!(context, "Idle wait errored: {:?}", err);
+                                }
+                            }
+                        }
+                        // if we can't properly terminate the idle
+                        // protocol let's break the connection.
+                        let res =
+                            async_std::future::timeout(Duration::from_secs(15), handle.done())
+                                .await
+                                .map_err(|err| {
+                                    self.trigger_reconnect();
+                                    Error::IdleTimeout(err)
+                                })?;
+
+                        match res {
+                            Ok(session) => {
+                                *self.session.lock().await = Some(Session::Secure(session));
+                            }
+                            Err(err) => {
+                                // if we cannot terminate IDLE it probably
+                                // means that we waited long (with idle_wait)
+                                // but the network went away/changed
+                                self.trigger_reconnect();
+                                return Err(Error::IdleProtocolFailed(err));
+                            }
+                        }
+                    }
+                    IdleHandle::Insecure(mut handle) => {
+                        handle.init().await?;
+
+                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
+                        *self.interrupt.lock().await = Some(interrupt);
+
+                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+                            // interrupt_idle has happened before we
+                            // provided self.interrupt
+                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
+                            std::mem::drop(idle_wait);
+                            info!(context, "Idle wait was skipped");
+                        } else {
+                            info!(context, "Idle entering wait-on-remote state");
+                            match idle_wait.await {
+                                Ok(IdleResponse::NewData(_)) => {
+                                    info!(context, "Idle has NewData");
+                                }
+                                // TODO: idle_wait does not distinguish manual interrupts
+                                // from Timeouts if we would know it's a Timeout we could bail
+                                // directly and reconnect .
+                                Ok(IdleResponse::Timeout) => {
+                                    info!(context, "Idle-wait timeout or interruption");
+                                }
+                                Ok(IdleResponse::ManualInterrupt) => {
+                                    info!(context, "Idle wait was interrupted");
+                                }
+                                Err(err) => {
+                                    warn!(context, "Idle wait errored: {:?}", err);
+                                }
+                            }
+                        }
+                        // if we can't properly terminate the idle
+                        // protocol let's break the connection.
+                        let res =
+                            async_std::future::timeout(Duration::from_secs(15), handle.done())
+                                .await
+                                .map_err(|err| {
+                                    self.trigger_reconnect();
+                                    Error::IdleTimeout(err)
+                                })?;
+
+                        match res {
+                            Ok(session) => {
+                                *self.session.lock().await = Some(Session::Insecure(session));
+                            }
+                            Err(err) => {
+                                // if we cannot terminate IDLE it probably
+                                // means that we waited long (with idle_wait)
+                                // but the network went away/changed
+                                self.trigger_reconnect();
+                                return Err(Error::IdleProtocolFailed(err));
+                            }
+                        }
+                    }
+                }
+            }
+
+            Ok(())
+        })
+    }
+
+    pub(crate) fn fake_idle(&self, context: &Context, watch_folder: Option<String>) {
+        // Idle using polling. This is also needed if we're not yet configured -
+        // in this case, we're waiting for a configure job (and an interrupt).
+        task::block_on(async move {
+            let fake_idle_start_time = SystemTime::now();
+
+            info!(context, "IMAP-fake-IDLEing...");
+
+            let interrupt = stop_token::StopSource::new();
+
+            // check every minute if there are new messages
+            // TODO: grow sleep durations / make them more flexible
+            let interval = async_std::stream::interval(Duration::from_secs(60));
+            let mut interrupt_interval = interrupt.stop_token().stop_stream(interval);
+            *self.interrupt.lock().await = Some(interrupt);
+            if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+                // interrupt_idle has happened before we
+                // provided self.interrupt
+                self.skip_next_idle_wait.store(false, Ordering::SeqCst);
+                info!(context, "fake-idle wait was skipped");
+            } else {
+                // loop until we are interrupted or if we fetched something
+                while let Some(_) = interrupt_interval.next().await {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context) {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.read().await.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break;
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
+
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break;
+                                }
+                            }
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
+                        }
+                    }
+                }
+            }
+            self.interrupt.lock().await.take();
+
+            info!(
+                context,
+                "IMAP-fake-IDLE done after {:.4}s",
+                SystemTime::now()
+                    .duration_since(fake_idle_start_time)
+                    .unwrap_or_default()
+                    .as_millis() as f64
+                    / 1000.,
+            );
+        })
+    }
+
+    pub fn interrupt_idle(&self, context: &Context) {
+        task::block_on(async move {
+            let mut interrupt: Option<stop_token::StopSource> = self.interrupt.lock().await.take();
+            if interrupt.is_none() {
+                // idle wait is not running, signal it needs to skip
+                self.skip_next_idle_wait.store(true, Ordering::SeqCst);
+
+                // meanwhile idle-wait may have produced the StopSource
+                interrupt = self.interrupt.lock().await.take();
+            }
+            // let's manually drop the StopSource
+            if interrupt.is_some() {
+                // the imap thread provided us a stop token but might
+                // not have entered idle_wait yet, give it some time
+                // for that to happen. XXX handle this without extra wait
+                // https://github.com/deltachat/deltachat-core-rust/issues/925
+                std::thread::sleep(Duration::from_millis(200));
+                info!(context, "low-level: dropping stop-source to interrupt idle");
+                std::mem::drop(interrupt)
+            }
+        });
+    }
+}

src/imap/select_folder.rs

@@ -0,0 +1,113 @@
+use super::Imap;
+
+use crate::context::Context;
+
+type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("IMAP Could not obtain imap-session object.")]
+    NoSession,
+
+    #[error("IMAP Connection Lost or no connection established")]
+    ConnectionLost,
+
+    #[error("IMAP Folder name invalid: {0}")]
+    BadFolderName(String),
+
+    #[error("IMAP close/expunge failed")]
+    CloseExpungeFailed(#[from] async_imap::error::Error),
+
+    #[error("IMAP other error: {0}")]
+    Other(String),
+}
+
+impl Imap {
+    /// select a folder, possibly update uid_validity and, if needed,
+    /// expunge the folder to remove delete-marked messages.
+    pub(super) async fn select_folder<S: AsRef<str>>(
+        &self,
+        context: &Context,
+        folder: Option<S>,
+    ) -> Result<()> {
+        if self.session.lock().await.is_none() {
+            let mut cfg = self.config.write().await;
+            cfg.selected_folder = None;
+            cfg.selected_folder_needs_expunge = false;
+            self.trigger_reconnect();
+            return Err(Error::NoSession);
+        }
+
+        // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
+        // if there is _no_ new folder, we continue as we might want to expunge below.
+        if let Some(ref folder) = folder {
+            if let Some(ref selected_folder) = self.config.read().await.selected_folder {
+                if folder.as_ref() == selected_folder {
+                    return Ok(());
+                }
+            }
+        }
+
+        // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
+        let needs_expunge = { self.config.read().await.selected_folder_needs_expunge };
+        if needs_expunge {
+            if let Some(ref folder) = self.config.read().await.selected_folder {
+                info!(context, "Expunge messages in \"{}\".", folder);
+
+                // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see
+                // https://tools.ietf.org/html/rfc3501#section-6.4.2
+                if let Some(ref mut session) = &mut *self.session.lock().await {
+                    match session.close().await {
+                        Ok(_) => {
+                            info!(context, "close/expunge succeeded");
+                        }
+                        Err(err) => {
+                            self.trigger_reconnect();
+                            return Err(Error::CloseExpungeFailed(err));
+                        }
+                    }
+                } else {
+                    return Err(Error::NoSession);
+                }
+            }
+            self.config.write().await.selected_folder_needs_expunge = false;
+        }
+
+        // select new folder
+        if let Some(ref folder) = folder {
+            if let Some(ref mut session) = &mut *self.session.lock().await {
+                let res = session.select(folder).await;
+
+                // https://tools.ietf.org/html/rfc3501#section-6.3.1
+                // says that if the server reports select failure we are in
+                // authenticated (not-select) state.
+
+                match res {
+                    Ok(mailbox) => {
+                        let mut config = self.config.write().await;
+                        config.selected_folder = Some(folder.as_ref().to_string());
+                        config.selected_mailbox = Some(mailbox);
+                        Ok(())
+                    }
+                    Err(async_imap::error::Error::ConnectionLost) => {
+                        self.trigger_reconnect();
+                        self.config.write().await.selected_folder = None;
+                        Err(Error::ConnectionLost)
+                    }
+                    Err(async_imap::error::Error::Validate(_)) => {
+                        Err(Error::BadFolderName(folder.as_ref().to_string()))
+                    }
+                    Err(err) => {
+                        self.config.write().await.selected_folder = None;
+                        self.trigger_reconnect();
+                        Err(Error::Other(err.to_string()))
+                    }
+                }
+            } else {
+                Err(Error::NoSession)
+            }
+        } else {
+            Ok(())
+        }
+    }
+}