remove one attr from mimeparser

They are now preserved as attachment, instead of diving into parsing-html
  and simplifying.

- adapt mime-debugging

.circleci/config.yml

@@ -7,10 +7,6 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
-  python:
-    docker:
-      - image: 3.7.7-stretch
-            
 
 restore-workspace: &restore-workspace
   attach_workspace:
@@ -40,6 +36,12 @@ jobs:
     executor: default
     steps:
       - checkout
+      - run:
+          name: Update submodules
+          command: git submodule update --init --recursive
+      - run:
+          name: Calculate dependencies
+          command: cargo generate-lockfile
       - restore_cache:
           keys:
             - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
@@ -47,6 +49,7 @@ jobs:
       - run: rustup default $(cat rust-toolchain)
       - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
       - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
+      - run: cargo update
       - run: cargo fetch
       - run: rustc +stable --version
       - run: rustc +$(cat rust-toolchain) --version
@@ -88,6 +91,7 @@ jobs:
             curl https://sh.rustup.rs -sSf | sh -s -- -y
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
+      - run: cargo update
       - run: cargo fetch
       - run:
           name: Test
@@ -124,19 +128,32 @@ jobs:
           paths:
             - c-docs
 
-  remote_python_packaging:
-    machine: true
-    steps:
-      - checkout
-      # the following commands on success produces
-      # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash ci_scripts/remote_python_packaging.sh 
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            # - c-docs
-            - py-docs
-            - wheelhouse
+  # build_test_docs_wheel:
+  #   docker:
+  #     - image: deltachat/coredeps
+  #       environment:
+  #           TESTS: 1
+  #           DOCS: 1
+  #   working_directory: /mnt/crate
+  #   steps:
+  #     - *restore-workspace
+  #     - *restore-cache
+  #     - run:
+  #         name: build docs, run tests and build wheels
+  #         command: ci_scripts/run-python.sh
+  #     - run:
+  #          name: copying docs and wheels to workspace
+  #          command: |
+  #            mkdir -p workspace/python
+  #            # cp -av docs workspace/c-docs
+  #            cp -av python/.docker-tox/wheelhouse workspace/
+  #            cp -av python/doc/_build/ workspace/py-docs
+  #     - persist_to_workspace:
+  #         root: workspace
+  #         paths:
+  #           # - c-docs
+  #           - py-docs
+  #           - wheelhouse
 
   remote_tests_rust:
     machine: true
@@ -148,7 +165,10 @@ jobs:
     machine: true
     steps:
       - checkout
+      #- attach_workspace:
+      #    at: workspace
       - run: ci_scripts/remote_tests_python.sh
+    # workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   upload_docs_wheels:
     machine: true
@@ -156,7 +176,6 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
-      - run: pyenv versions
       - run: pyenv global 3.5.2
       - run: ls -laR workspace
       - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
@@ -168,7 +187,7 @@ jobs:
       - *restore-cache
       - run:
           name: Run cargo clippy
-          command: cargo clippy 
+          command: cargo clippy --all
 
 
 workflows:
@@ -176,44 +195,24 @@ workflows:
 
   test:
     jobs:
-      # - cargo_fetch
+      - cargo_fetch
 
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
+      - remote_tests_rust
 
-      - remote_tests_python:
-          filters:
-            tags:
-              only: /.*/
+      - remote_tests_python
 
-      - remote_python_packaging:
-          requires: 
-             - remote_tests_python
-             - remote_tests_rust 
-          filters:
-            tags:
-              only: /.*/
-
-      - upload_docs_wheels:
+      # - upload_docs_wheels:
+      #     requires:
+      #       - build_test_docs_wheel
+      #       - build_doxygen
+      - rustfmt:
           requires:
-             - remote_python_packaging
-             - build_doxygen
-          filters:
-            tags:
-              only: /.*/
-      # - rustfmt:
-      #     requires:
-      #       - cargo_fetch
-      # - clippy:
-      #     requires:
-      #      - cargo_fetch
+            - cargo_fetch
+      - clippy:
+          requires:
+           - cargo_fetch
 
-      - build_doxygen:
-          filters:
-            tags:
-              only: /.*/
+      - build_doxygen
 
       # Linux Desktop 64bit
       # - test_x86_64-unknown-linux-gnu:

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-12
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-12
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-12
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

CHANGELOG.md

@@ -1,326 +1,8 @@
 # Changelog 
 
-## 1.33.0
-
-- let `dc_set_muted()` also mute one-to-one chats #1470
-
-- fix a bug that led to load and traffic if the server does not use sent-folder
-  #1472
-
-
-## 1.32.0
-
-- fix endless loop when trying to download messages with bad RFC Message-ID,
-  also be more reliable on similar errors #1463 #1466 #1462
-
-- fix bug with comma in contact request #1438
-
-- do not refer to hidden messages on replies #1459
-
-- improve error handling #1468 #1465 #1464
-
-
-## 1.31.0
-
-- always describe the context of the displayed error #1451
-
-- do not emit `DC_EVENT_ERROR` when message sending fails;
-  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
-
-- new config-option `media_quality` #1449
-
-- try over if writing message to database fails #1447
-
-
-## 1.30.0
-
-- expunge deleted messages #1440
-
-- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
-
-
-## 1.29.0
-
-- new config options `delete_device_after` and `delete_server_after`,
-  each taking an amount of seconds after which messages
-  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
-
-- new api `dc_estimate_deletion_cnt()` to estimate the effect
-  of `delete_device_after` and `delete_server_after`
-
-- use Ed25519 keys by default, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1362
-
-- improve message ellipsizing #1397 #1430
-
-- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
-
-- do not show badly formatted non-delta-messages as empty #1384
-
-- try over SMTP on potentially recoverable error 5.5.0 #1379
-
-- remove device-chat from forward-to-chat-list #1367
-
-- improve group-handling #1368
-
-- `dc_get_info()` returns uptime (how long the context is in use)
-
-- python improvements and adaptions #1408 #1415
-
-- log to the stdout and stderr in tests #1416
-
-- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
-
-- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
-
-- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
-
-
-## 1.28.0
-
-- new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()
-  that will sort the "saved messages" chat to the top of the chatlist #1336
-- mark mails as being deleted from server in dc_empty_server() #1333
-- fix interaction with servers that do not allow folder creation on root-level;
-  use path separator as defined by the email server #1359
-- fix group creation if group was created by non-delta clients #1357
-- fix showing replies from non-delta clients #1353
-- fix member list on rejoining left groups #1343
-- fix crash when using empty groups #1354
-- fix potential crash on special names #1350
-
-
-## 1.27.0
-
-- handle keys reliably on armv7 #1327
-
-
-## 1.26.0
-
-- change generated key type back to RSA as shipped versions
-  have problems to encrypt to Ed25519 keys
-
-- update rPGP to encrypt reliably to Ed25519 keys;
-  one of the next versions can finally use Ed25519 keys then
-
-
-## 1.25.0
-
-- save traffic by downloading only messages that are really displayed #1236
-
-- change generated key type to Ed25519, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1287
-
-- improve key handling #1237 #1240 #1242 #1247
-
-- mute handling, apis are dc_set_chat_mute_duration()
-  dc_chat_is_muted() and dc_chat_get_remaining_mute_duration() #1143
-
-- pinning chats, new apis are dc_set_chat_visibility() and
-  dc_chat_get_visibility() #1248
-
-- add dc_provider_new_from_email() api that queries the new, integrated
-  provider-database #1207
-
-- account creation by scanning a qr code
-  in the DCACCOUNT scheme (https://mailadm.readthedocs.io),
-  new api is dc_set_config_from_qr() #1249
-
-- if possible, dc_join_securejoin(), returns the new chat-id immediately
-  and does the handshake in background #1225
-
-- update imap and smtp dependencies #1115
-
-- check for MOVE capability before using MOVE command #1263
-
-- allow inline attachments from RFC 2183 #1280
-
-- fix updating names from incoming mails #1298
-
-- fix error messages shown on import #1234
-
-- directly attempt to re-connect if the smtp connection is maybe stale #1296
-
-- improve adding group members #1291
-
-- improve rust-api #1261
-
-- cleanup #1302 #1283 #1282 #1276 #1270-#1274 #1267 #1258-#1260
-  #1257 #1239 #1231 #1224
-
-- update spec #1286 #1291
-
-
-## 1.0.0-beta.24
-
-- fix oauth2/gmail bug introduced in beta23 (not used in releases) #1219
-
-- fix panic when receiving eg. cyrillic filenames #1216
-
-- delete all consumed secure-join handshake messagess #1209 #1212
-
-- rust-level cleanups #1218 #1217 #1210 #1205
-
-- python-level cleanups #1204 #1202 #1201
-
-
-## 1.0.0-beta.23
-
-- #1197 fix imap-deletion of messages 
-
-- #1171 Combine multiple MDNs into a single mail, reducing traffic 
-
-- #1155 fix to not send out gossip always, reducing traffic
-
-- #1160 fix reply-to-encrypted determination 
-
-- #1182 Add "Auto-Submitted: auto-replied" header to MDNs
-
-- #1194 produce python wheels again, fix c/py.delta.chat
-  master-deployment 
-
-- rust-level housekeeping and improvements #1161 #1186 #1185 #1190 #1194 #1199 #1191 #1190 #1184 and more
-
-- #1063 clarify licensing 
-
-- #1147 use mailparse 0.10.2 
-
-
-## 1.0.0-beta.22
-
-- #1095 normalize email lineends to CRLF
-
-- #1095 enable link-time-optimization, saves eg. on android 11 mb
-
-- #1099 fix import regarding devicechats
-
-- #1092 improve logging
-
-- #1096 #1097 #1094 #1090 #1091 internal cleanups
-
-## 1.0.0-beta.21
-
-- #1078 #1082 ensure RFC compliance by producing 78 column lines for
-  encoded attachments. 
-
-- #1080 don't recreate and thus break group membership if an unknown 
-  sender (or mailer-daemon) sends a message referencing the group chat 
-
-- #1081 #1079 some internal cleanups 
-
-- update imap-proto dependency, to fix yandex/oauth 
-
-## 1.0.0-beta.20
-
-- #1074 fix OAUTH2/gmail
-- #1072 fix group members not appearing in contact list
-- #1071 never block interrupt_idle (thus hopefully also not on maybe_network())
-- #1069 reduce smtp-timeout to 30 seconds
-- #1066 #1065 avoid unwrap in dehtml, make literals more readable
-
-## 1.0.0-beta.19
-
-- #1058 timeout smtp-send if it doesn't complete in 15 minutes 
-
-- #1059 trim down logging
-
-## 1.0.0-beta.18
-
-- #1056 avoid panicking when we couldn't read imap-server's greeting
-  message 
-
-- #1055 avoid panicking when we don't have a selected folder
-
-- #1052 #1049 #1051 improve logging to add thread-id/name and
-  file/lineno to each info/warn message.
-
-- #1050 allow python bindings to initialize Account with "os_name".
-
-
-## 1.0.0-beta.17
-
-- #1044 implement avatar recoding to 192x192 in core to keep file sizes small. 
-
-- #1024 fix #1021 SQL/injection malformed Chat-Group-Name breakage
-
-- #1036 fix smtp crash by pulling in a fixed async-smtp 
-
-- #1039 fix read-receipts appearing as normal messages when you change
-  MDN settings 
-
-- #1040 do not panic on SystemTimeDifference
-
-- #1043 avoid potential crashes in malformed From/Chat-Disposition... headers  
-
-- #1045 #1041 #1038 #1035 #1034 #1029 #1025 various cleanups and doc
-  improvments 
-
-## 1.0.0-beta.16
-
-- alleviate login problems with providers which only
-  support RSA1024 keys by switching back from Rustls 
-  to native-tls, by using the new async-email/async-native-tls 
-  crate from @dignifiedquire. thanks @link2xt. 
-
-- introduce per-contact profile images to send out 
-  own profile image heuristically, and fix sending
-  out of profile images in "in-prepare" groups. 
-  this also extends the Chat-spec that is maintained
-  in core to specify Chat-Group-Image and Chat-Group-Avatar
-  headers. thanks @r10s and @hpk42.
-
-- fix merging of protected headers from the encrypted
-  to the unencrypted parts, now not happening recursively
-  anymore.  thanks @hpk and @r10s
-
-- fix/optimize autocrypt gossip headers to only get 
-  sent when there are more than 2 people in a chat. 
-  thanks @link2xt
-
-- fix displayname to use the authenticated name 
-  when available (displayname as coming from contacts 
-  themselves). thanks @simon-laux
-
-- introduce preliminary support for offline autoconfig 
-  for nauta provider. thanks @hpk42 @r10s
-
-## 1.0.0-beta.15
-
-- fix #994 attachment appeared doubled in chats (and where actually
-  downloaded after smtp-send). @hpk42
-
-## 1.0.0-beta.14
-
-- fix packaging issue with our rust-email fork, now we are tracking
-  master again there. hpk42
-
-## 1.0.0-beta.13
-
-- fix #976 -- unicode-issues in display-name of email addresses. @hpk42
-
-- fix #985 group add/remove member bugs resulting in broken groups.  @hpk42
-
-- fix hanging IMAP connections -- we now detect with a 15second timeout
-  if we cannot terminate the IDLE IMAP protocol. @hpk42 @link2xt
-
-- fix incoming multipart/mixed containing html, to show up as
-  attachments again.  Fixes usage for simplebot which sends html
-  files for users to interact with the bot. @adbenitez @hpk42 
-
-- refinements to internal autocrypt-handling code, do not send
-  prefer-encrypt=nopreference as it is the default if no attribute
-  is present.  @linkxt 
-
-- simplify, modularize and rustify several parts 
-  of dc-core (general WIP). @link2xt @flub @hpk42 @r10s
-
-- use async-email/async-smtp to handle SMTP connections, might
-  fix connection/reconnection issues. @link2xt 
-
-- more tests and refinements for dealing with blobstorage @flub @hpk42 
-
-- use a dedicated build-server for CI testing of core PRs
+## next (pending)
 
+- restructured (but not change) imap idle handling into own file. cc @link2xt
 
 ## 1.0.0-beta.12
 

Cargo.toml

@@ -1,42 +1,41 @@
 [package]
 name = "deltachat"
-version = "1.33.0"
+version = "1.0.0-beta.12"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
-license = "MPL-2.0"
-
-[profile.release]
-lto = true
+license = "MPL"
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { git = "https://github.com/rpgp/rpgp", branch = "master", default-features = false }
 hex = "0.4.0"
 sha2 = "0.8.0"
-rand = "0.7.0"
-smallvec = "1.0.0"
-reqwest = { version = "0.10.0", features = ["blocking", "json"] }
-num-derive = "0.3.0"
+rand = "0.6.5"
+smallvec = "0.6.9"
+reqwest = { version = "0.9.15", default-features = false, features = ["rustls-tls"] }
+num-derive = "0.2.5"
 num-traits = "0.2.6"
-async-smtp = "0.2"
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.2"
-async-native-tls = "0.3.1"
-async-std = { version = "1.4", features = ["unstable"] }
+lettre = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
+async-imap = { git = "https://github.com/async-email/async-imap", branch="master" }
+async-tls = "0.6"
+async-std = { version = "1.0", features = ["unstable"] }
 base64 = "0.11"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
-indexmap = "1.3.0"
+failure = "0.1.5"
+failure_derive = "0.1.5"
+# TODO: make optional
+rustyline = "4.1.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.21", features = ["bundled"] }
-r2d2_sqlite = "0.13.0"
+rusqlite = { version = "0.20", features = ["bundled"] }
+r2d2_sqlite = "0.12.0"
 r2d2 = "0.8.5"
 strum = "0.16.0"
 strum_macros = "0.16.0"
@@ -45,21 +44,17 @@ backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.17.1"
+quick-xml = "0.15.0"
 escaper = "0.1.0"
 bitflags = "1.1.0"
 debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.0"
+rustls = "0.16.0"
+webpki-roots = "0.18.0"
+webpki = "0.21.0"
+mailparse = "0.10.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
-pretty_env_logger = "0.3.1"
-
-rustyline = { version = "4.1.0", optional = true }
-thiserror = "1.0.14"
-anyhow = "1.0.28"
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -80,11 +75,10 @@ path = "examples/simple.rs"
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
-required-features = ["rustyline"]
 
 
 [features]
-default = ["nightly"]
-vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
+default = ["nightly", "ringbuf"]
+vendored = []
 nightly = ["pgp/nightly"]
-
+ringbuf = ["pgp/ringbuf"]

LICENSE

@@ -2,6 +2,9 @@ The files in this directory and under its subdirectories
 are (c) 2019 by Bjoern Petersen and contributors and released under the
 Mozilla Public License Version 2.0, see below for a copy.
 
+NOTE that the files in the "libs" directory are copyrighted by third parties
+and come with their own respective licenses.
+
 Mozilla Public License Version 2.0
 ==================================
 

README.md

@@ -17,9 +17,8 @@ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-cargo run --example repl -- ~/deltachat-db
+cargo run --example repl -- /path/to/db
 ```
-where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 
@@ -109,27 +108,9 @@ $ cargo test -- --ignored
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
+- `ringbuf`: Enable the use of [`slice_deque`](https://github.com/gnzlbg/slice_deque) in pgp.
 
 [circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
 [circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
 [appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
 [appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
-
-## Language bindings and frontend projects
-
-Language bindings are available for:
-
-- [C](https://c.delta.chat)
-- [Node.js](https://www.npmjs.com/package/deltachat-node)
-- [Python](https://py.delta.chat)
-- [Go](https://github.com/hugot/go-deltachat/)
-- **Java** and **Swift** (contained in the Android/iOS repos)
-
-The following "frontend" projects make use of the Rust-library
-or its language bindings:
-
-- [Android](https://github.com/deltachat/deltachat-android)
-- [iOS](https://github.com/deltachat/deltachat-ios)
-- [Desktop](https://github.com/deltachat/deltachat-desktop)
-- [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
-- several **Bots**

appveyor.yml

@@ -4,10 +4,11 @@ environment:
 
 install:
   - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-12
+  - rustup-init -yv --default-toolchain nightly-2019-07-10
   - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
   - rustc -vV
   - cargo -vV
+  - cargo update
 
 build: false
 

ci_scripts/ci_upload.sh

@@ -14,22 +14,24 @@ DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
 
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
+# DISABLED: python docs to py.delta.chat
+#ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+#rsync -avz \
+#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+#  "$PYDOCDIR/html/" \
+#  delta@py.delta.chat:build/${BRANCH}
 
 # C docs to c.delta.chat
 ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
 rsync -avz \
-  --delete \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   "$DOXYDOCDIR/html/" \
   delta@c.delta.chat:build-c/${BRANCH}
 
+exit 0
+
+# OUTDATED -- for re-use from python release-scripts 
+
 echo -----------------------
 echo upload wheels 
 echo -----------------------
@@ -37,7 +39,6 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip setuptools
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN
@@ -49,9 +50,6 @@ devpi use dc/$N_BRANCH || {
     devpi use dc/$N_BRANCH
 }
 devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*
+devpi upload deltachat*.whl
 
 popd
-
-# remove devpi non-master dc indices if thy are too old
-python ci_scripts/cleanup_devpi_indices.py

ci_scripts/cleanup_devpi_indices.py

@@ -1,72 +0,0 @@
-"""
-Remove old "dc" indices except for master which always stays.
-
-"""
-from requests import Session
-import datetime
-import sys
-import subprocess
-
-MAXDAYS=7
-
-session = Session()
-session.headers["Accept"] = "application/json"
-
-
-def get_indexes(baseurl, username):
-    response = session.get(baseurl + username)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    return result["indexes"]
-
-
-def get_projectnames(baseurl, username, indexname):
-    response = session.get(baseurl + username + "/" + indexname)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    return result["projects"]
-
-
-def get_release_dates(baseurl, username, indexname, projectname):
-    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    dates = set()
-    for value in result.values():
-        if "+links" not in value:
-            continue
-        for link in value["+links"]:
-            for log in link["log"]:
-                dates.add(tuple(log["when"]))
-    return dates
-
-
-def run():
-    baseurl = "https://m.devpi.net/"
-    username = "dc"
-    for indexname in get_indexes(baseurl, username):
-        projectnames = get_projectnames(baseurl, username, indexname)
-        if indexname == "master" or not indexname:
-            continue
-        clear_index = not projectnames
-        for projectname in projectnames:
-            dates = get_release_dates(baseurl, username, indexname, projectname)
-            if not dates:
-                print(
-                    "%s has no releases" % (baseurl + username + "/" + indexname),
-                    file=sys.stderr)
-                date = datetime.datetime.now()
-            else:
-                date = datetime.datetime(*max(dates))
-            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
-                assert username and indexname
-                clear_index = True
-                break
-        if clear_index:
-            url = baseurl + username + "/" + indexname
-            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
-
-
-
-if __name__ == '__main__':
-    run()

ci_scripts/docker-coredeps/Dockerfile

@@ -5,11 +5,14 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate 
+# Install a recent Perl, needed to install OpenSSL
 ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
 
+# Install OpenSSL
+ADD deps/build_openssl.sh /builder/build_openssl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
+
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # Install python tools (auditwheels,tox, ...)

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,6 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
-
-# remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library
+#
+set -e -x
+
+# perform clean build of core and install 
+export TOXWORKDIR=.docker-tox
+
+# build core library
+
+cargo build --release -p deltachat_ffi
+
+# configure access to a base python and 
+# to several python interpreters needed by tox below
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+#
+# run python tests
+#
+
+if [ -n "$TESTS" ]; then 
+
+    echo ----------------
+    echo run python tests
+    echo ----------------
+
+    pushd python 
+    # first run all tests ...
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
+    popd
+fi
+
+
+if [ -n "$DOCS" ]; then 
+    echo -----------------------
+    echo generating python docs
+    echo -----------------------
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+fi

ci_scripts/old/run-python.sh

@@ -46,7 +46,6 @@ if [ -n "$TESTS" ]; then
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
     unset DCC_PY_LIVECONFIG
-    unset DCC_NEW_TMP_EMAIL
     tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
     tox --workdir "$TOXWORKDIR" -e auditwheels
     popd

ci_scripts/remote_python_packaging.sh

@@ -1,51 +0,0 @@
-#!/bin/bash 
-
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-set -xe
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist 
-# we seem to need .git for setuptools_scm versioning 
-find .git >>.rsynclist
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-
-set +x
-
-# we have to create a remote file for the remote-docker run 
-# so we can do a simple ssh command with a TTY
-# so that when our job dies, all container-runs are aborted.
-# sidenote: the circle-ci machinery will kill ongoing jobs
-# if there are new commits and we want to ensure that 
-# everything is terminated/cleaned up and we have no orphaned
-# useless still-running docker-containers consuming resources. 
-
-ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
-    set +x -e
-    cd $BUILDDIR
-    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
-    set -x
-
-    # run everything else inside docker 
-    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
-
-_HERE
-
-echo "--- Running $CIRCLE_JOB remotely"
-
-ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
-mkdir -p workspace 
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux1*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -30,7 +30,6 @@ ssh $SSHTARGET <<_HERE
     export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
     #rm -rf virtualenv venv

ci_scripts/run-doxygen.sh

@@ -3,4 +3,5 @@
 set -ex
 
 cd deltachat-ffi 
-PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 
+doxygen 
+

ci_scripts/run-python-test.sh

@@ -6,7 +6,7 @@
 set -e -x
 
 # for core-building and python install step
-export DCC_RS_TARGET=debug
+export DCC_RS_TARGET=release
 export DCC_RS_DEV=`pwd`
 
 cd python

ci_scripts/run_all.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat Core Rust library, Python wheels and docs 
-
-set -e -x
-
-# Perform clean build of core and install.
-export TOXWORKDIR=.docker-tox
-
-# compile core lib
-
-export PATH=/root/.cargo/bin:$PATH
-cargo build --release -p deltachat_ffi
-# cargo test --all --all-features
-
-# Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
-
-# Configure access to a base python and to several python interpreters
-# needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-ln -s /opt/python/cp38-cp38/bin/python3.8
-popd
-
-pushd python
-# prepare a clean tox run
-rm -rf tests/__pycache__
-rm -rf src/deltachat/__pycache__
-mkdir -p $TOXWORKDIR
-
-# disable live-account testing to speed up test runs and wheel building
-# XXX we may switch on some live-tests on for better ensurances 
-# Note that the independent remote_tests_python step does all kinds of
-# live-testing already. 
-unset DCC_PY_LIVECONFIG
-unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
-popd
-
-
-echo -----------------------
-echo generating python docs
-echo -----------------------
-(cd python && tox --workdir "$TOXWORKDIR" -e doc)

deltachat-ffi/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "deltachat_ffi"
-version = "1.33.0"
+version = "1.0.0-beta.12"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
-license = "MPL-2.0"
+license = "MIT OR Apache-2.0"
 
 keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
@@ -16,14 +16,14 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { path = "../", default-features = false }
+deltachat-provider-database = "0.2.1"
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
-serde_json = "1.0"
-anyhow = "1.0.28"
-thiserror = "1.0.14"
+failure = "0.1.6"
 
 [features]
-default = ["vendored", "nightly"]
+default = ["vendored", "nightly", "ringbuf"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
+ringbuf = ["deltachat/ringbuf"]

deltachat-ffi/src/providers.rs

@@ -0,0 +1,93 @@
+extern crate deltachat_provider_database;
+
+use std::ptr;
+
+use crate::string::{to_string_lossy, StrExt};
+use deltachat_provider_database::StatusState;
+
+#[no_mangle]
+pub type dc_provider_t = deltachat_provider_database::Provider;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_domain(
+    domain: *const libc::c_char,
+) -> *const dc_provider_t {
+    match deltachat_provider_database::get_provider_info(&to_string_lossy(domain)) {
+        Some(provider) => provider,
+        None => ptr::null(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_email(
+    email: *const libc::c_char,
+) -> *const dc_provider_t {
+    let email = to_string_lossy(email);
+    let domain = deltachat_provider_database::get_domain_from_email(&email);
+    match deltachat_provider_database::get_provider_info(domain) {
+        Some(provider) => provider,
+        None => ptr::null(),
+    }
+}
+
+macro_rules! null_guard {
+    ($context:tt) => {
+        if $context.is_null() {
+            return ptr::null_mut() as *mut libc::c_char;
+        }
+    };
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_overview_page(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    format!(
+        "{}/{}",
+        deltachat_provider_database::PROVIDER_OVERVIEW_URL,
+        (*provider).overview_page
+    )
+    .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_name(provider: *const dc_provider_t) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).name.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_markdown(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).markdown.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_status_date(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).status.date.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_status(provider: *const dc_provider_t) -> u32 {
+    if provider.is_null() {
+        return 0;
+    }
+    match (*provider).status.state {
+        StatusState::OK => 1,
+        StatusState::PREPARATION => 2,
+        StatusState::BROKEN => 3,
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_unref(_provider: *const dc_provider_t) {
+    ()
+}
+
+// TODO expose general provider overview url?

deltachat-ffi/src/string.rs

@@ -1,5 +1,5 @@
+use failure::Fail;
 use std::ffi::{CStr, CString};
-use std::ptr;
 
 /// Duplicates a string
 ///
@@ -8,7 +8,7 @@ use std::ptr;
 /// # Examples
 ///
 /// ```rust,norun
-/// use crate::string::{dc_strdup, to_string_lossy};
+/// use deltachat::dc_tools::{dc_strdup, to_string_lossy};
 /// unsafe {
 ///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
 ///     let str_a_copy = dc_strdup(str_a);
@@ -16,7 +16,7 @@ use std::ptr;
 ///     assert_ne!(str_a, str_a_copy);
 /// }
 /// ```
-unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
+pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
     let ret: *mut libc::c_char;
     if !s.is_null() {
         ret = libc::strdup(s);
@@ -30,13 +30,13 @@ unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
 }
 
 /// Error type for the [OsStrExt] trait
-#[derive(Debug, PartialEq, thiserror::Error)]
-pub(crate) enum CStringError {
+#[derive(Debug, Fail, PartialEq)]
+pub enum CStringError {
     /// The string contains an interior null byte
-    #[error("String contains an interior null byte")]
+    #[fail(display = "String contains an interior null byte")]
     InteriorNullByte,
     /// The string is not valid Unicode
-    #[error("String is not valid unicode")]
+    #[fail(display = "String is not valid unicode")]
     NotUnicode,
 }
 
@@ -65,7 +65,7 @@ pub(crate) enum CStringError {
 ///     let mut c_ptr: *mut libc::c_char = dc_strdup(path_c.as_ptr());
 /// }
 /// ```
-pub(crate) trait OsStrExt {
+pub trait OsStrExt {
     /// Convert a  [std::ffi::OsStr] to an [std::ffi::CString]
     ///
     /// This is useful to convert e.g. a [std::path::Path] to
@@ -130,28 +130,27 @@ fn os_str_to_c_string_unicode(
 }
 
 /// Convenience methods/associated functions for working with [CString]
-trait CStringExt {
-    /// Create a new [CString], best effort
+///
+/// This is helps transitioning from unsafe code.
+pub trait CStringExt {
+    /// Create a new [CString], yolo style
     ///
-    /// Like the [to_string_lossy] this doesn't give up in the face of
-    /// bad input (embedded null bytes in this case) instead it does
-    /// the best it can by stripping the embedded null bytes.
-    fn new_lossy<T: Into<Vec<u8>>>(t: T) -> CString {
-        let mut s = t.into();
-        s.retain(|&c| c != 0);
-        CString::new(s).unwrap_or_default()
+    /// This unwrap the result, panicking when there are embedded NULL
+    /// bytes.
+    fn yolo<T: Into<Vec<u8>>>(t: T) -> CString {
+        CString::new(t).expect("String contains null byte, can not be CString")
     }
 }
 
 impl CStringExt for CString {}
 
-/// Convenience methods to turn strings into C strings.
+/// Convenience methods to make transitioning from raw C strings easier.
 ///
 /// To interact with (legacy) C APIs we often need to convert from
 /// Rust strings to raw C strings.  This can be clumsy to do correctly
 /// and the compiler sometimes allows it in an unsafe way.  These
 /// methods make it more succinct and help you get it right.
-pub(crate) trait Strdup {
+pub trait StrExt {
     /// Allocate a new raw C `*char` version of this string.
     ///
     /// This allocates a new raw C string which must be freed using
@@ -168,52 +167,14 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl<T: AsRef<str>> Strdup for T {
+impl<T: AsRef<str>> StrExt for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self.as_ref());
+        let tmp = CString::yolo(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-// We can not implement for AsRef<OsStr> because we already implement
-// AsRev<str> and this conflicts.  So implement for Path directly.
-impl Strdup for std::path::Path {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
-/// Convenience methods to turn optional strings into C strings.
-///
-/// This is the same as the [Strdup] trait but a different trait name
-/// to work around the type system not allowing to implement [Strdup]
-/// for `Option<impl Strdup>` When we already have an [Strdup] impl
-/// for `AsRef<&str>`.
-///
-/// When the [Option] is [Option::Some] this behaves just like
-/// [Strdup::strdup], when it is [Option::None] a null pointer is
-/// returned.
-pub(crate) trait OptStrdup {
-    /// Allocate a new raw C `*char` version of this string, or NULL.
-    ///
-    /// See [Strdup::strdup] for details.
-    unsafe fn strdup(&self) -> *mut libc::c_char;
-}
-
-impl<T: AsRef<str>> OptStrdup for Option<T> {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        match self {
-            Some(s) => {
-                let tmp = CString::new_lossy(s.as_ref());
-                dc_strdup(tmp.as_ptr())
-            }
-            None => ptr::null_mut(),
-        }
-    }
-}
-
-pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
+pub fn to_string_lossy(s: *const libc::c_char) -> String {
     if s.is_null() {
         return "".into();
     }
@@ -223,7 +184,7 @@ pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
     cstr.to_string_lossy().to_string()
 }
 
-pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
+pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
     if s.is_null() {
         return None;
     }
@@ -244,7 +205,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 ///
 /// [Path]: std::path::Path
 #[cfg(not(target_os = "windows"))]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     assert!(!s.is_null(), "cannot be used on null pointers");
     use std::os::unix::ffi::OsStrExt;
     unsafe {
@@ -256,7 +217,7 @@ pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 
 // as_path() implementation for windows, documented above.
 #[cfg(target_os = "windows")]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     as_path_unicode(s)
 }
 
@@ -363,14 +324,8 @@ mod tests {
     }
 
     #[test]
-    fn test_cstring_new_lossy() {
-        assert!(CString::new("hel\x00lo").is_err());
-        assert!(CString::new(String::from("hel\x00o")).is_err());
-        let r = CString::new("hello").unwrap();
-        assert_eq!(CString::new_lossy("hello"), r);
-        assert_eq!(CString::new_lossy("hel\x00lo"), r);
-        assert_eq!(CString::new_lossy(String::from("hello")), r);
-        assert_eq!(CString::new_lossy(String::from("hel\x00lo")), r);
+    fn test_cstring_yolo() {
+        assert_eq!(CString::new("hello").unwrap(), CString::yolo("hello"));
     }
 
     #[test]
@@ -392,19 +347,4 @@ mod tests {
             assert_eq!(cmp, 0);
         }
     }
-
-    #[test]
-    fn test_strdup_opt_string() {
-        unsafe {
-            let s = Some("hello");
-            let c = s.strdup();
-            let cmp = strcmp(c, b"hello\x00" as *const u8 as *const libc::c_char);
-            free(c as *mut libc::c_void);
-            assert_eq!(cmp, 0);
-
-            let s: Option<&str> = None;
-            let c = s.strdup();
-            assert_eq!(c, ptr::null_mut());
-        }
-    }
 }

deltachat_derive/Cargo.toml

@@ -1,13 +1,12 @@
 [package]
 name = "deltachat_derive"
-version = "2.0.0"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+version = "0.1.0"
+authors = ["Dmitry Bogatov <KAction@debian.org>"]
 edition = "2018"
-license = "MPL-2.0"
 
 [lib]
 proc-macro = true
 
 [dependencies]
-syn = "1.0.13"
-quote = "1.0.2"
+syn = "0.14.4"
+quote = "0.6.3"

deltachat_derive/src/lib.rs

@@ -3,6 +3,7 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
+use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to

draft/group-sync.rst

@@ -1,126 +0,0 @@
-
-Problem: missing eventual group consistency
---------------------------------------------
-
-If group members are concurrently adding new members,
-the new members will miss each other's additions, example:
-
-- Alice and Bob are in a two-member group
-
-- Alice adds Carol, concurrently Bob adds Doris
-
-- Carol will see a three-member group (Alice, Bob, Carol),
-  Doris will see a different three-member group (Alice, Bob, Doris),
-  and only Alice and Bob will have all four members.
-
-Note that for verified groups any mitigation mechanism likely
-needs to make all clients to know who originally added a member.
-
-
-solution: memorize+attach (possible encrypted) chat-meta mime messages
-----------------------------------------------------------------------
-
-For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
-
-
-- All Chat-Group-Member-Added/Removed messages are recorded in their
-  full raw (signed and encrypted) mime-format in the DB
-
-- If an incoming member-add/member-delete messages has a member list
-  which is, apart from the added/removed member, not consistent
-  with our own view, broadcast a "Chat-Group-Member-Correction" message to
-  all members, attaching the original added/removed mime-message for all mismatching
-  contacts.  If we have no relevant add/del information, don't send a
-  correction message out.
-
-- Upong receiving added/removed attachments we don't do the
-  check_consistency+correction message dance.
-  This avoids recursion problems and hard-to-reason-about chatter.
-
-Notes:
-
-- mechanism works for both encrypted and unencrypted add/del messages
-
-- we already have a "mime_headers" column in the DB for each incoming message.
-  We could extend it to also include the payload and store mime unconditionally
-  for member-added/removed messages.
-
-- multiple member-added/removed messages can be attached in a single
-  correction message
-
-- it is minimal on the number of overall messages to reach group consistency
-  (best-case: no extra messages, the ABCD case above: max two extra messages)
-
-- somewhat backward compatible: older clients will probably ignore
-  messages which are signed by someone who is not the outer From-address.
-
-- the correction-protocol also helps with dropped messages.  If a member
-  did not see a member-added/removed message, the next member add/removed
-  message in the group will likely heal group consistency for this member.
-
-- we can quite easily extend the mechanism to also provide the group-avatar or
-  other meta-information.
-
-Discussions of variants
-++++++++++++++++++++++++
-
-- instead of acting on MemberAdded/Removed message we could send
-  corrections for any received message that addresses inconsistent group members but
-  a) this would delay group-membership healing
-  b) could lead to a lot of members sending corrections
-
-- instead of broadcasting correction messages we could only send it to
-  the sender of the inconsistent member-added/removed message.
-  A receiver of such a correction message would then need to forward
-  the message to the members it thinks also have an inconsistent view.
-  This sounds complicated and error-prone.  Concretely, if Alice
-  receives Bob's "Member-added: Doris" message, then Alice
-  broadcasting the correction message with "Member-added: Carol"
-  would reach all four members, healing group consistency in one step.
-  If Bob meanwhile receives Alice's "Member-Added: Carol" message,
-  Bob would broadcast a correction message to all four members as well.
-  (Imagine a situation where Alice/Bob added Carol/Doris
-  while both being in an offline or bad-connection situation).
-
-
-solution2: repeat member-added/removed messages
----------------------------------------------------
-
-Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
-but keep sending out the old headers until the new protocol is sufficiently deployed.
-
-The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
-which controls repetition of the signed "add/del e-mail address" payload.
-
-Example::
-
-    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
-        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
-        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
-        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
-        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
-        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
-        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
-        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
-        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
-        =P6GG
-
-TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
-Receivers will apply the add/del change to the group-membership,
-decrease the TTL by 1, and if TTL>0 re-sent the header.
-
-The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
-This allows to propagate, in a cryptographically secured way,
-who added a member. This is particularly important for allowing
-to show in verified groups who added a member (planned).
-
-Disadvantage to solution 1:
-
-- requires to specify encoding and precise rules for what/how is signed.
-
-- causes O(N^2) extra messages
-
-- Not easily extendable for other things (without introducing a new
-  header / encoding)
-
-

examples/repl/cmdline.rs

@@ -1,9 +1,9 @@
 use std::path::Path;
 use std::str::FromStr;
 
-use anyhow::{bail, ensure};
-use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
+use deltachat::chat::{self, Chat};
 use deltachat::chatlist::*;
+use deltachat::config;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
@@ -19,16 +19,15 @@ use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::Event;
-use deltachat::{config, provider};
 
-/// Reset database tables.
+/// Reset database tables. This function is called from Core cmdline.
 /// Argument is a bitmask, executing single or multiple actions in one call.
 /// e.g. bitmask 7 triggers actions definded with bits 1, 2 and 4.
-fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
-    println!("Resetting tables ({})...", bits);
+pub fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
+    info!(context, "Resetting tables ({})...", bits);
     if 0 != bits & 1 {
         sql::execute(context, &context.sql, "DELETE FROM jobs;", params![]).unwrap();
-        println!("(1) Jobs reset.");
+        info!(context, "(1) Jobs reset.");
     }
     if 0 != bits & 2 {
         sql::execute(
@@ -38,11 +37,11 @@ fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
             params![],
         )
         .unwrap();
-        println!("(2) Peerstates reset.");
+        info!(context, "(2) Peerstates reset.");
     }
     if 0 != bits & 4 {
         sql::execute(context, &context.sql, "DELETE FROM keypairs;", params![]).unwrap();
-        println!("(4) Private keypairs reset.");
+        info!(context, "(4) Private keypairs reset.");
     }
     if 0 != bits & 8 {
         sql::execute(
@@ -81,23 +80,21 @@ fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
         )
         .unwrap();
         sql::execute(context, &context.sql, "DELETE FROM leftgrps;", params![]).unwrap();
-        println!("(8) Rest but server config reset.");
+        info!(context, "(8) Rest but server config reset.");
     }
 
     context.call_cb(Event::MsgsChanged {
-        chat_id: ChatId::new(0),
+        chat_id: 0,
         msg_id: MsgId::new(0),
     });
 
     1
 }
 
-fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
+fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), Error> {
     let data = dc_read_file(context, filename)?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false) {
-        println!("dc_receive_imf errored: {:?}", err);
-    }
+    dc_receive_imf(context, &data, "import", 0, 0);
     Ok(())
 }
 
@@ -156,7 +153,7 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
                 let name = name_f.to_string_lossy();
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
-                    println!("Import: {}", path_plus_name);
+                    info!(context, "Import: {}", path_plus_name);
                     if dc_poke_eml_file(context, path_plus_name).is_ok() {
                         read_cnt += 1
                     }
@@ -164,10 +161,13 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
             }
         }
     }
-    println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
+    info!(
+        context,
+        "Import: {} items read from \"{}\".", read_cnt, &real_spec
+    );
     if read_cnt > 0 {
         context.call_cb(Event::MsgsChanged {
-            chat_id: ChatId::new(0),
+            chat_id: 0,
             msg_id: MsgId::new(0),
         });
     }
@@ -188,7 +188,8 @@ fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     };
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
-    println!(
+    info!(
+        context,
         "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
@@ -222,14 +223,16 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id.is_daymarker() {
-            println!(
+            info!(
+                context,
                 "--------------------------------------------------------------------------------"
             );
 
             lines_out += 1
         } else if !msg_id.is_special() {
             if lines_out == 0 {
-                println!(
+                info!(
+                    context,
                     "--------------------------------------------------------------------------------",
                 );
                 lines_out += 1
@@ -239,7 +242,8 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
         }
     }
     if lines_out > 0 {
-        println!(
+        info!(
+            context,
             "--------------------------------------------------------------------------------"
         );
     }
@@ -289,7 +293,7 @@ fn log_contactlist(context: &Context, contacts: &Vec<u32>) {
                 );
             }
 
-            println!("Contact#{}: {}{}", contact_id, line, line2);
+            info!(context, "Contact#{}: {}{}", contact_id, line, line2);
         }
     }
 }
@@ -298,9 +302,9 @@ fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
+pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
     let chat_id = *context.cmdline_sel_chat_id.read().unwrap();
-    let mut sel_chat = if !chat_id.is_unset() {
+    let mut sel_chat = if chat_id > 0 {
         Chat::load_from_db(context, chat_id).ok()
     } else {
         None
@@ -372,8 +376,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  listmedia\n\
                  archive <chat-id>\n\
                  unarchive <chat-id>\n\
-                 pin <chat-id>\n\
-                 unpin <chat-id>\n\
                  delchat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
@@ -395,10 +397,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
-                 providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
-                 estimatedeletion <seconds>\n\
                  emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
@@ -489,7 +489,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             println!("{:#?}", context.get_info());
         }
         "interrupt" => {
-            interrupt_inbox_idle(context);
+            interrupt_inbox_idle(context, true);
         }
         "maybenetwork" => {
             maybe_network(context);
@@ -508,26 +508,23 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             let cnt = chatlist.len();
             if cnt > 0 {
-                println!(
+                info!(
+                    context,
                     "================================================================================"
                 );
 
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
-                    println!(
-                        "{}#{}: {} [{} fresh] {}",
+                    info!(
+                        context,
+                        "{}#{}: {} [{} fresh]",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat.get_id().get_fresh_msg_cnt(context),
-                        match chat.visibility {
-                            ChatVisibility::Normal => "",
-                            ChatVisibility::Archived => "📦",
-                            ChatVisibility::Pinned => "📌",
-                        },
+                        chat::get_fresh_msg_cnt(context, chat.get_id()),
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
-                    let statestr = if chat.visibility == ChatVisibility::Archived {
+                    let statestr = if chat.is_archived() {
                         " [Archived]"
                     } else {
                         match lot.get_state() {
@@ -541,7 +538,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     let timestr = dc_timestamp_to_str(lot.get_timestamp());
                     let text1 = lot.get_text1();
                     let text2 = lot.get_text2();
-                    println!(
+                    info!(
+                        context,
                         "{}{}{}{} [{}]{}",
                         text1.unwrap_or(""),
                         if text1.is_some() { ": " } else { "" },
@@ -554,13 +552,14 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                             ""
                         },
                     );
-                    println!(
+                    info!(
+                        context,
                         "================================================================================"
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(context, ChatId::new(0)) {
-                println!("Location streaming enabled.");
+            if location::is_sending_locations_to_chat(context, 0) {
+                info!(context, "Location streaming enabled.");
             }
             println!("{} chats", cnt);
         }
@@ -569,8 +568,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 bail!("Argument [chat-id] is missing.");
             }
             if !arg1.is_empty() {
-                let chat_id = ChatId::new(arg1.parse()?);
-                println!("Selecting chat {}", chat_id);
+                let chat_id = arg1.parse()?;
+                println!("Selecting chat #{}", chat_id);
                 sel_chat = Some(Chat::load_from_db(context, chat_id)?);
                 *context.cmdline_sel_chat_id.write().unwrap() = chat_id;
             }
@@ -588,7 +587,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             } else {
                 format!("{} member(s)", members.len())
             };
-            println!(
+            info!(
+                context,
                 "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
@@ -608,11 +608,14 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 },
             );
             log_msglist(context, &msglist)?;
-            if let Some(draft) = sel_chat.get_id().get_draft(context)? {
+            if let Some(draft) = chat::get_draft(context, sel_chat.get_id())? {
                 log_msg(context, "Draft", &draft);
             }
 
-            println!("{} messages.", sel_chat.get_id().get_msg_cnt(context));
+            println!(
+                "{} messages.",
+                chat::get_msg_cnt(context, sel_chat.get_id())
+            );
             chat::marknoticed_chat(context, sel_chat.get_id())?;
         }
         "createchat" => {
@@ -688,7 +691,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let contacts = chat::get_chat_contacts(context, sel_chat.as_ref().unwrap().get_id());
-            println!("Memberlist:");
+            info!(context, "Memberlist:");
 
             log_contactlist(context, &contacts);
             println!(
@@ -714,7 +717,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
-                println!(
+                info!(
+                    context,
                     "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
                     dc_timestamp_to_str(location.timestamp),
@@ -728,7 +732,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 );
             }
             if locations.is_empty() {
-                println!("No locations.");
+                info!(context, "No locations.");
             }
         }
         "sendlocations" => {
@@ -794,7 +798,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             let chat = if let Some(ref sel_chat) = sel_chat {
                 sel_chat.get_id()
             } else {
-                ChatId::new(0)
+                0 as libc::c_uint
             };
 
             let msglist = context.search_msgs(chat, arg1);
@@ -808,14 +812,14 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             if !arg1.is_empty() {
                 let mut draft = Message::new(Viewtype::Text);
                 draft.set_text(Some(arg1.to_string()));
-                sel_chat
-                    .as_ref()
-                    .unwrap()
-                    .get_id()
-                    .set_draft(context, Some(&mut draft));
+                chat::set_draft(
+                    context,
+                    sel_chat.as_ref().unwrap().get_id(),
+                    Some(&mut draft),
+                );
                 println!("Draft saved.");
             } else {
-                sel_chat.as_ref().unwrap().get_id().set_draft(context, None);
+                chat::set_draft(context, sel_chat.as_ref().unwrap().get_id(), None);
                 println!("Draft deleted.");
             }
         }
@@ -851,23 +855,15 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             }
             print!("\n");
         }
-        "archive" | "unarchive" | "pin" | "unpin" => {
+        "archive" | "unarchive" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.set_visibility(
-                context,
-                match arg0 {
-                    "archive" => ChatVisibility::Archived,
-                    "unarchive" | "unpin" => ChatVisibility::Normal,
-                    "pin" => ChatVisibility::Pinned,
-                    _ => panic!("Unexpected command (This should never happen)"),
-                },
-            )?;
+            let chat_id = arg1.parse()?;
+            chat::archive(context, chat_id, arg0 == "archive")?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.delete(context)?;
+            let chat_id = arg1.parse()?;
+            chat::delete(context, chat_id)?;
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -888,7 +884,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             );
 
             let mut msg_ids = [MsgId::new(0); 1];
-            let chat_id = ChatId::new(arg2.parse()?);
+            let chat_id = arg2.parse()?;
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(context, &msg_ids, chat_id)?;
         }
@@ -940,14 +936,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             let contact = Contact::get_by_id(context, contact_id)?;
             let name_n_addr = contact.get_name_n_addr();
 
-            let mut res = format!(
-                "Contact info for: {}:\nIcon: {}\n",
-                name_n_addr,
-                match contact.get_profile_image(context) {
-                    Some(image) => image.to_str().unwrap().to_string(),
-                    None => "NoIcon".to_string(),
-                }
-            );
+            let mut res = format!("Contact info for: {}:\n\n", name_n_addr);
 
             res += &Contact::get_encrinfo(context, contact_id)?;
 
@@ -984,31 +973,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 res.get_text2()
             );
         }
-        "setqr" => {
-            ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            match set_config_from_qr(context, arg1) {
-                Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
-                Err(err) => println!("Cannot set config from QR code: {:?}", err),
-            }
-        }
-        "providerinfo" => {
-            ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1) {
-                Some(info) => {
-                    println!("Information for provider belonging to {}:", arg1);
-                    println!("status: {}", info.status as u32);
-                    println!("before_login_hint: {}", info.before_login_hint);
-                    println!("after_login_hint: {}", info.after_login_hint);
-                    println!("overview_page: {}", info.overview_page);
-                    for server in info.server.iter() {
-                        println!("server: {}:{}", server.hostname, server.port,);
-                    }
-                }
-                None => {
-                    println!("No information for provider belonging to {} found.", arg1);
-                }
-            }
-        }
         // TODO: implement this again, unclear how to match this through though, without writing a parser.
         // "event" => {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
@@ -1030,16 +994,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 bail!("Command failed.");
             }
         }
-        "estimatedeletion" => {
-            ensure!(!arg1.is_empty(), "Argument <seconds> missing");
-            let seconds = arg1.parse()?;
-            let device_cnt = message::estimate_deletion_cnt(context, false, seconds)?;
-            let server_cnt = message::estimate_deletion_cnt(context, true, seconds)?;
-            println!(
-                "estimated count of messages older than {} seconds:\non device: {}\non server: {}",
-                seconds, device_cnt, server_cnt
-            );
-        }
         "emptyserver" => {
             ensure!(!arg1.is_empty(), "Argument <flags> missing");
 

examples/repl/main.rs

@@ -7,6 +7,8 @@
 #[macro_use]
 extern crate deltachat;
 #[macro_use]
+extern crate failure;
+#[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate rusqlite;
@@ -18,9 +20,8 @@ use std::process::Command;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 
-use anyhow::{bail, Error};
-use deltachat::chat::ChatId;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::context::*;
 use deltachat::job::*;
 use deltachat::oauth2::*;
@@ -40,7 +41,7 @@ use self::cmdline::*;
 
 // Event Handler
 
-fn receive_event(_context: &Context, event: Event) {
+fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
     match event {
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
@@ -110,6 +111,8 @@ fn receive_event(_context: &Context, event: Event) {
             print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
         }
     }
+
+    0
 }
 
 // Threads for waiting for messages and for jobs
@@ -199,7 +202,7 @@ fn stop_threads(context: &Context) {
         println!("Stopping threads");
         IS_RUNNING.store(false, Ordering::Relaxed);
 
-        interrupt_inbox_idle(context);
+        interrupt_inbox_idle(context, true);
         interrupt_mvbox_idle(context);
         interrupt_sentbox_idle(context);
         interrupt_smtp_idle(context);
@@ -261,7 +264,7 @@ const DB_COMMANDS: [&str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 24] = [
     "listchats",
     "listarchived",
     "chat",
@@ -285,8 +288,6 @@ const CHAT_COMMANDS: [&str; 26] = [
     "listmedia",
     "archive",
     "unarchive",
-    "pin",
-    "unpin",
     "delchat",
 ];
 const MESSAGE_COMMANDS: [&str; 8] = [
@@ -307,17 +308,8 @@ const CONTACT_COMMANDS: [&str; 6] = [
     "delcontact",
     "cleanupcontacts",
 ];
-const MISC_COMMANDS: [&str; 10] = [
-    "getqr",
-    "getbadqr",
-    "checkqr",
-    "event",
-    "fileinfo",
-    "clear",
-    "exit",
-    "quit",
-    "help",
-    "estimatedeletion",
+const MISC_COMMANDS: [&str; 9] = [
+    "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
 ];
 
 impl Hinter for DcHelper {
@@ -369,10 +361,10 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-fn main_0(args: Vec<String>) -> Result<(), Error> {
+fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
-        bail!("No db-name specified");
+        return Err(format_err!("No db-name specified"));
     }
     let context = Context::new(
         Box::new(receive_event),
@@ -442,7 +434,7 @@ enum ExitResult {
     Exit,
 }
 
-fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
+fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failure::Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
@@ -470,7 +462,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
         }
         "configure" => {
             start_threads(ctx.clone());
-            ctx.read().unwrap().configure();
+            configure(&ctx.read().unwrap());
         }
         "oauth2" => {
             if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
@@ -494,10 +486,9 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
         }
         "getqr" | "getbadqr" => {
             start_threads(ctx.clone());
-            if let Some(mut qr) = dc_get_securejoin_qr(
-                &ctx.read().unwrap(),
-                ChatId::new(arg1.parse().unwrap_or_default()),
-            ) {
+            if let Some(mut qr) =
+                dc_get_securejoin_qr(&ctx.read().unwrap(), arg1.parse().unwrap_or_default())
+            {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -525,7 +516,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
     Ok(ExitResult::Continue)
 }
 
-pub fn main() -> Result<(), Error> {
+pub fn main() -> Result<(), failure::Error> {
     let _ = pretty_env_logger::try_init();
 
     let args: Vec<String> = std::env::args().collect();

examples/simple.rs

@@ -7,6 +7,7 @@ use tempfile::tempdir;
 use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::job::{
@@ -15,18 +16,21 @@ use deltachat::job::{
 };
 use deltachat::Event;
 
-fn cb(_ctx: &Context, event: Event) {
+fn cb(_ctx: &Context, event: Event) -> usize {
     print!("[{:?}]", event);
 
     match event {
         Event::ConfigureProgress(progress) => {
-            println!("  progress: {}", progress);
+            print!("  progress: {}\n", progress);
+            0
         }
         Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            println!("  {}", msg);
+            print!("  {}\n", msg);
+            0
         }
         _ => {
-            println!();
+            print!("\n");
+            0
         }
     }
 }
@@ -76,7 +80,7 @@ fn main() {
     ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
         .unwrap();
     ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-    ctx.configure();
+    configure(&ctx);
 
     thread::sleep(duration);
 
@@ -100,7 +104,7 @@ fn main() {
     println!("stopping threads");
 
     *running.write().unwrap() = false;
-    deltachat::job::interrupt_inbox_idle(&ctx);
+    deltachat::job::interrupt_inbox_idle(&ctx, true);
     deltachat::job::interrupt_smtp_idle(&ctx);
 
     println!("joining");

python/CHANGELOG

@@ -1,29 +1,6 @@
-0.900.0 (DRAFT)
----------------
-
-- refactored internals to use plugin-approach 
-
-- introduced PerAccount and Global hooks that plugins can implement 
-
-- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
-
-- introduced two documented examples for an echo and a group-membership
-  tracking plugin. 
-
-0.800.0
--------
-
-- use latest core 1.25.0 
-
-- refine tests and some internal changes to core bindings
-
-0.700.0
+0.600.1
 ---------
 
-- lots of new Python APIs
-
-- use rust core-beta23 
-
 - introduce automatic versioning via setuptools_scm, 
   based on py-X.Y.Z tags 
 

python/README.rst

@@ -3,98 +3,41 @@ deltachat python bindings
 =========================
 
 This package provides bindings to the deltachat-core_ Rust -library
-which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
-a low-level Chat/Contact/Message API to user interfaces and bots.
+which provides imap/smtp/crypto handling as well as chat/group/messages
+handling to Android, Desktop and IO user interfaces.
 
+Installing pre-built packages (linux-only)
+==========================================
 
-Installing bindings from source (Updated: 20-Jan-2020)
-=========================================================
-
-Install Rust and Cargo first.  Deltachat needs a specific nightly
-version, the easiest is probably to first install Rust stable from
-rustup and then use this to install the correct nightly version.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then GIT clone the deltachat-core-rust repo and get the actual
-rust- and cargo-toolchain needed by deltachat::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-   rustup show
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   ./install_python_bindings.py
-
-The installation might take a while, depending on your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_NEW_TMP_EMAIL`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs. This is implemented on the server side via
-  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
-Installing pre-built packages (Linux-only)
-========================================================
-
-If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
+If you have a linux system you may install the ``deltachat`` binary "wheel" package
 without any "build-from-source" steps.
 
-We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
-then create a fresh Python virtual environment and activate it in your shell::
+1. `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+   then create a fresh python environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
         source venv/bin/activate
 
-Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``venv`` directory and leaves
-your system installation alone.
+   Afterwards, invoking ``python`` or ``pip install`` will only
+   modify files in your ``venv`` directory and leave your system installation
+   alone.
+
+2. Install the wheel for linux::
+
+        pip install deltachat
+
+    Verify it worked by typing::
+
+        python -c "import deltachat"
+
+
+Installing a wheel from a PR/branch
+---------------------------------------
 
 For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest
-github ``master`` branch::
+and push them to a python package index. To install the latest github ``master`` branch::
 
-    pip install --pre -i https://m.devpi.net/dc/master deltachat
-
-To verify it worked::
-
-    python -c "import deltachat"
+    pip install -i https://m.devpi.net/dc/master deltachat
 
 .. note::
 
@@ -103,6 +46,65 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
+Installing bindings from source
+===============================
+
+If you can't use "binary" method above then you need to compile
+to core deltachat library::
+
+    git clone https://github.com/deltachat/deltachat-core-rust
+    cd deltachat-core-rust
+    cd python
+
+If you don't have one active, create and activate a python "virtualenv":
+
+    python virtualenv venv  # or python -m venv
+    source venv/bin/activate
+
+Afterwards ``which python`` tells you that it comes out of the "venv"
+directory that contains all python install artifacts. Let's first
+install test tools::
+
+    pip install pytest pytest-timeout pytest-rerunfailures requests
+
+then cargo-build and install the deltachat bindings::
+
+    python install_python_bindings.py
+
+The bindings will be installed in release mode but with debug symbols.
+The release mode is necessary because some tests generate RSA keys
+which is prohibitively slow in debug mode.
+
+After successful binding installation you can finally run the tests::
+
+    pytest -v tests
+
+.. note::
+
+    Some tests are sometimes failing/hanging because of
+    https://github.com/deltachat/deltachat-core-rust/issues/331
+    and
+    https://github.com/deltachat/deltachat-core-rust/issues/326
+
+
+running "live" tests (experimental)
+-----------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_PY_LIVECONFIG`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
+
 Code examples
 =============
 
@@ -113,11 +115,15 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux1 based wheels
-================================
+Building manylinux1 wheels
+==========================
+
+.. note::
+
+   This section may not fully work.
 
 Building portable manylinux1 wheels which come with libdeltachat.so
-can be done with docker-tooling.
+and all it's dependencies is easy using the provided docker tooling.
 
 using docker pull / premade images
 ------------------------------------
@@ -130,9 +136,9 @@ organization::
 
 This docker image can be used to run tests and build Python wheels for all interpreters::
 
-    $ docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+    $ bash ci_scripts/ci_run.sh
+
+This command runs tests and build-wheel scripts in a docker container.
 
 
 Optionally build your own docker image

python/doc/_static/custom.css

@@ -15,7 +15,3 @@ div.globaltoc {
 img.logo {
     height: 120px;
 }
-
-div.footer {
-    display: none;
-}

python/doc/api.rst

@@ -2,6 +2,10 @@
 high level API reference
 ========================
 
+.. note::
+
+    This API is work in progress and may change in versions prior to 1.0.
+
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
 - :class:`deltachat.contact.Contact`

python/doc/capi.rst

@@ -0,0 +1,7 @@
+
+C deltachat interface
+=====================
+
+See :doc:`lapi` for accessing many of the below functions
+through the ``deltachat.capi.lib`` namespace.
+

python/doc/conf.py

@@ -55,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'deltachat'
-copyright = u'2020, holger krekel and contributors'
+copyright = u'2018, holger krekel and contributors'
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation

python/doc/examples.rst

@@ -1,60 +1,37 @@
 
+
 examples
 ========
 
+
+Playing around on the commandline
+----------------------------------
+
 Once you have :doc:`installed deltachat bindings <install>`
-you need email/password credentials for an IMAP/SMTP account.
-Delta Chat developers and the CI system use a special URL to create
-temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
+you can start playing from the python interpreter commandline.
+For example you can type ``python`` and then::
 
-Receiving a Chat message from the command line
-----------------------------------------------
+    # instantiate and configure deltachat account
+    import deltachat
+    ac = deltachat.Account("/tmp/db")
 
-Here is a simple bot that:
+    # start configuration activity and smtp/imap threads
+    ac.start_threads()
+    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
 
-- receives a message and sends back ("echoes") a message
+    # create a contact and send a message
+    contact = ac.create_contact("someother@email.address")
+    chat = ac.create_chat_by_contact(contact)
+    chat.send_text("hi from the python interpreter command line")
 
-- terminates the bot if the message `/quit` is sent
+Checkout our :doc:`api` for the various high-level things you can do
+to send/receive messages, create contacts and chats.
 
-.. include:: ../examples/echo_and_quit.py
-    :literal:
 
-With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    $ cd examples
-    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
-
-While this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Track member additions and removals in a group
-----------------------------------------------
-
-Here is a simple bot that:
-
-- echoes messages sent to it
-
-- tracks if configuration completed
-
-- tracks member additions and removals for all chat groups
-
-.. include:: ../examples/group_tracking.py
-    :literal:
-
-With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
-
-When this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Writing bots for real
+Looking at a real example
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a little framework for writing deltachat bots in Python.
+contains a real-life example of Python bindings usage.
+
 

python/doc/index.rst

@@ -1,15 +1,15 @@
 deltachat python bindings
 =========================
 
-The ``deltachat`` Python package provides two layers of bindings for the
-core Rust-library of the https://delta.chat messaging ecosystem:
+The ``deltachat`` Python package provides two bindings for the core Rust-library
+of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core.
+- :doc:`api` is a high level interface to deltachat-core which aims
+  to be memory safe and thoroughly tested through continous tox/pytest runs.
 
-- :doc:`plugins` is a brief introduction into implementing plugin hooks.
-
-- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
-  <https://github.com/deltachat/deltachat-core-rust>`_.
+- :doc:`capi` is a lowlevel CFFI-binding to the previous
+  `deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
+  replicates exactly the same C-level API).
 
 
 
@@ -28,8 +28,8 @@ getting started
    links
    changelog
    api
+   capi
    lapi
-   plugins
 
 ..
     Indices and tables

python/doc/plugins.rst

@@ -1,38 +0,0 @@
-
-Implementing Plugin Hooks
-==========================
-
-The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
-for managing global and per-account plugin registration, and performing
-hook calls. There are two kinds of plugins:
-
-- Global plugins that are active for all accounts; they can implement
-  hooks at account-creation and account-shutdown time.
-
-- Account plugins that are only active during the lifetime of a
-  single Account instance.
-
-
-Registering a plugin
---------------------
-
-.. autofunction:: deltachat.register_global_plugin
-    :noindex:
-
-.. automethod:: deltachat.account.Account.add_account_plugin
-    :noindex:
-
-
-Per-Account Hook specifications
--------------------------------
-
-.. autoclass:: deltachat.hookspec.PerAccount
-    :members:
-
-
-Global Hook specifications
---------------------------
-
-.. autoclass:: deltachat.hookspec.Global
-    :members:
-

python/examples/echo_and_quit.py

@@ -1,30 +0,0 @@
-
-# content of echo_and_quit.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class EchoPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.accept_sender_contact()
-            addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_message_delivered(self, message):
-        print("ac_message_delivered", message)
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/group_tracking.py

@@ -1,52 +0,0 @@
-
-# content of group_tracking.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class GroupTrackingPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.accept_sender_contact()
-            addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_outgoing_message(self, message):
-        print("ac_outgoing_message:", message)
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        print("ac_configure_completed:", success)
-
-    @account_hookimpl
-    def ac_chat_modified(self, chat):
-        print("ac_chat_modified:", chat.id, chat.get_name())
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/test_examples.py

@@ -1,72 +0,0 @@
-
-import pytest
-import py
-import echo_and_quit
-import group_tracking
-from deltachat.eventlogger import FFIEventLogger
-
-
-@pytest.fixture(scope='session')
-def datadir():
-    """The py.path.local object of the test-data/ directory."""
-    for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
-        if datadir.isdir():
-            return datadir
-    else:
-        pytest.skip('test-data directory not found')
-
-
-def test_echo_quit_plugin(acfactory):
-    botproc = acfactory.run_bot_process(echo_and_quit)
-
-    ac1 = acfactory.get_one_online_account()
-    bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert "hello" in reply.text
-    assert reply.chat == ch1
-    ch1.send_text("/quit")
-    botproc.wait()
-
-
-def test_group_tracking_plugin(acfactory, lp):
-    lp.sec("creating one group-tracking bot and two temp accounts")
-    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
-
-    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
-
-    botproc.fnmatch_lines("""
-        *ac_configure_completed*
-    """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
-
-    lp.sec("creating bot test group with bot")
-    bot_contact = ac1.create_contact(botproc.addr)
-    ch = ac1.create_group_chat("bot test group")
-    ch.add_contact(bot_contact)
-    ch.send_text("hello")
-
-    botproc.fnmatch_lines("""
-        *ac_chat_modified*bot test group*
-    """)
-
-    lp.sec("adding third member {}".format(ac2.get_config("addr")))
-    contact3 = ac1.create_contact(ac2.get_config("addr"))
-    ch.add_contact(contact3)
-
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert "hello" in reply.text
-
-    lp.sec("now looking at what the bot received")
-    botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
-
-    lp.sec("contact successfully added, now removing")
-    ch.remove_contact(contact3)
-    botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))

python/install_python_bindings.py

@@ -11,15 +11,18 @@ import sys
 if __name__ == "__main__":
     target = os.environ.get("DCC_RS_TARGET")
     if target is None:
-        os.environ["DCC_RS_TARGET"] = target = "debug"
+        os.environ["DCC_RS_TARGET"] = target = "release"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
-    cmd = ["cargo", "build", "-p", "deltachat_ffi"]
-    if target == 'release':
-        cmd.append("--release")
-    subprocess.check_call(cmd)
+    # build the core library  in release + debug mode because
+    # as of Nov 2019 rPGP generates RSA keys which take
+    # prohibitively long for non-release installs
+    os.environ["RUSTFLAGS"] = "-g"
+    subprocess.check_call([
+        "cargo", "build", "-p", "deltachat_ffi", "--" + target
+    ])
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":

python/setup.py

@@ -13,20 +13,14 @@ def main():
             "root": "..",
             "relative_to": __file__,
             'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
-            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
         },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'six'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
-        entry_points = {
-            'pytest11': [
-                'deltachat.testplugin = deltachat.testplugin',
-            ],
-        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,13 +1,6 @@
-import sys
-
-from . import capi, const, hookspec
-from .capi import ffi
-from .account import Account  # noqa
-from .message import Message  # noqa
-from .contact import Contact  # noqa
-from .chat import Chat        # noqa
-from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import eventlogger
+from deltachat import capi, const
+from deltachat.capi import ffi
+from deltachat.account import Account  # noqa
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -81,62 +74,3 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
-
-
-def register_global_plugin(plugin):
-    """ Register a global plugin which implements one or more
-    of the :class:`deltachat.hookspec.Global` hooks.
-    """
-    gm = hookspec.Global._get_plugin_manager()
-    gm.register(plugin)
-    gm.check_pending()
-
-
-def unregister_global_plugin(plugin):
-    gm = hookspec.Global._get_plugin_manager()
-    gm.unregister(plugin)
-
-
-register_global_plugin(eventlogger)
-
-
-def run_cmdline(argv=None, account_plugins=None):
-    """ Run a simple default command line app, registering the specified
-    account plugins. """
-    import argparse
-    if argv is None:
-        argv = sys.argv
-
-    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
-    parser.add_argument("db", action="store", help="database file")
-    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
-    parser.add_argument("--email", action="store", help="email address")
-    parser.add_argument("--password", action="store", help="password")
-
-    args = parser.parse_args(argv[1:])
-
-    ac = Account(args.db)
-
-    if args.show_ffi:
-        log = eventlogger.FFIEventLogger(ac, "bot")
-        ac.add_account_plugin(log)
-
-    if not ac.is_configured():
-        assert args.email and args.password, (
-            "you must specify --email and --password once to configure this database/account"
-        )
-        ac.set_config("addr", args.email)
-        ac.set_config("mail_pw", args.password)
-        ac.set_config("mvbox_move", "0")
-        ac.set_config("mvbox_watch", "0")
-        ac.set_config("sentbox_watch", "0")
-
-    for plugin in account_plugins or []:
-        ac.add_account_plugin(plugin)
-
-    # start IO threads and configure if neccessary
-    ac.start()
-
-    print("{}: waiting for message".format(ac.get_config("addr")))
-
-    ac.wait_shutdown()

python/src/deltachat/account.py

@@ -2,25 +2,22 @@
 
 from __future__ import print_function
 import atexit
-from contextlib import contextmanager
-from email.utils import parseaddr
-import queue
-from threading import Event
-import os
+import threading
+import re
+import time
 from array import array
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty
+
 import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message, map_system_message
+from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker
-from . import hookspec, iothreads
-
-
-class MissingCredentials(ValueError):
-    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
@@ -28,63 +25,38 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    MissingCredentials = MissingCredentials
-
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param os_name: this will be put to the X-Mailer header in outgoing messages
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        :param eventlogging: if False no eventlogging and no context callback will be configured
+        :param debug: turn on debug logging for events.
         """
-        # initialize per-account plugin system
-        self._pm = hookspec.PerAccount._make_plugin_manager()
-        self._logging = logging
-        self.add_account_plugin(self)
-
         self._dc_context = ffi.gc(
-            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
+            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
             _destroy_dc_context,
         )
+        if eventlogging:
+            self._evlogger = EventLogger(self._dc_context, logid, debug)
+            deltachat.set_context_callback(self._dc_context, self._process_event)
+            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
+        else:
+            self._threads = IOThreads(self._dc_context)
 
-        hook = hookspec.Global._get_plugin_manager().hook
-
-        self._threads = iothreads.IOThreads(self)
-        self._hook_event_queue = queue.Queue()
-        self._in_use_iter_events = False
-        self._shutdown_event = Event()
-
-        # open database
-        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
+        self._imex_events = Queue()
         atexit.register(self.shutdown)
-        hook.dc_account_init(account=self)
-
-    def disable_logging(self):
-        """ disable logging. """
-        self._logging = False
-
-    def enable_logging(self):
-        """ re-enable logging. """
-        self._logging = True
-
-    @hookspec.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        for name, kwargs in self._map_ffi_event(ffi_event):
-            ev = HookEvent(self, name=name, kwargs=kwargs)
-            self._hook_event_queue.put(ev)
 
     # def __del__(self):
     #    self.shutdown()
 
-    def ac_log_line(self, msg):
-        if self._logging:
-            self._pm.hook.ac_log_line(message=msg)
-
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -122,12 +94,9 @@ class Account(object):
         """
         self._check_config_key(name)
         name = name.encode("utf8")
+        value = value.encode("utf8")
         if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
-        if value is not None:
-            value = value.encode("utf8")
-        else:
-            value = ffi.NULL
         lib.dc_set_config(self._dc_context, name, value)
 
     def get_config(self, name):
@@ -144,27 +113,16 @@ class Account(object):
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr, public, secret):
-        """See dc_preconfigure_keypair() in deltachat.h.
-
-        In other words, you don't need this.
-        """
-        res = lib.dc_preconfigure_keypair(self._dc_context,
-                                          as_dc_charpointer(addr),
-                                          as_dc_charpointer(public),
-                                          as_dc_charpointer(secret))
-        if res == 0:
-            raise Exception("Failed to set key")
-
-    def update_config(self, kwargs):
-        """ update config values.
+    def configure(self, **kwargs):
+        """ set config values and configure this account.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for key, value in kwargs.items():
-            self.set_config(key, str(value))
+        for name, value in kwargs.items():
+            self.set_config(name, value)
+        lib.dc_configure(self._dc_context)
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -172,19 +130,7 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return bool(lib.dc_is_configured(self._dc_context))
-
-    def set_avatar(self, img_path):
-        """Set self avatar.
-
-        :raises ValueError: if profile image could not be set
-        :returns: None
-        """
-        if img_path is None:
-            self.set_config("selfavatar", None)
-        else:
-            assert os.path.exists(img_path), img_path
-            self.set_config("selfavatar", img_path)
+        return lib.dc_is_configured(self._dc_context)
 
     def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
@@ -202,6 +148,11 @@ class Account(object):
             raise ValueError("no flags set")
         lib.dc_empty_server(self._dc_context, flags)
 
+    def get_infostring(self):
+        """ return info of the configured account. """
+        self.check_is_configured()
+        return from_dc_charpointer(lib.dc_get_info(self._dc_context))
+
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -223,7 +174,8 @@ class Account(object):
 
         :returns: :class:`deltachat.contact.Contact`
         """
-        return Contact(self, const.DC_CONTACT_ID_SELF)
+        self.check_is_configured()
+        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
 
     def create_contact(self, email, name=None):
         """ create a (new) Contact. If there already is a Contact
@@ -234,14 +186,11 @@ class Account(object):
         :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
+        name = as_dc_charpointer(name)
+        email = as_dc_charpointer(email)
+        contact_id = lib.dc_create_contact(self._dc_context, name, email)
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
-        return Contact(self, contact_id)
+        return Contact(self._dc_context, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -254,14 +203,6 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
-        _, addr = parseaddr(email)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
-        if contact_id:
-            return self.get_contact_by_id(contact_id)
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -281,15 +222,7 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
-    def get_fresh_messages(self):
-        """ yield all fresh messages from all chats. """
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
-        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
+        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
 
     def create_chat_by_contact(self, contact):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
@@ -311,9 +244,6 @@ class Account(object):
         """ create or get an existing chat object for the
         the specified message.
 
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
         :param message: messsage id or message instance.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
@@ -366,13 +296,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -417,37 +340,45 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory.
-
-        Note that the account does not have to be started.
-        """
+        """ export public and private keys to the specified directory. """
         return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
-
-        Note that the account does not have to be started.
         """
         export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
+    def _imex_events_clear(self):
+        try:
+            while True:
+                self._imex_events.get_nowait()
+        except Empty:
+            pass
+
     def _export(self, path, imex_cmd):
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
-            return imex_tracker.wait_finish()
+        self._imex_events_clear()
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        files_written = []
+        while True:
+            ev = self._imex_events.get()
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif isinstance(ev, bool):
+                if not ev:
+                    raise ValueError("export failed, exp-files: {}".format(files_written))
+                return files_written
 
     def import_self_keys(self, path):
         """ Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
-
-        Note that the account does not have to be started.
         """
         self._import(path, imex_cmd=2)
 
@@ -460,11 +391,12 @@ class Account(object):
         self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
-            imex_tracker.wait_finish()
+        self._imex_events_clear()
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        if not self._imex_events.get():
+            raise ValueError("import from path '{}' failed".format(path))
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -529,6 +461,63 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
+    def stop_ongoing(self):
+        lib.dc_stop_ongoing_process(self._dc_context)
+
+    #
+    # meta API for start/stop and event based processing
+    #
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.get_message_by_id(ev[2])
+
+    def start_threads(self, mvbox=False, sentbox=False):
+        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
+
+        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
+        :returns: None
+        """
+        if not self.is_configured():
+            self.configure()
+        self._threads.start(mvbox=mvbox, sentbox=sentbox)
+
+    def stop_threads(self, wait=True):
+        """ stop IMAP/SMTP threads. """
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=wait)
+
+    def shutdown(self, wait=True):
+        """ stop threads and close and remove underlying dc_context and callbacks. """
+        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
+            # print("SHUTDOWN", self)
+            self.stop_threads(wait=False)
+            lib.dc_close(self._dc_context)
+            self.stop_threads(wait=wait)  # to wait for threads
+            deltachat.clear_context_callback(self._dc_context)
+            del self._dc_context
+            atexit.unregister(self.shutdown)
+
+    def _process_event(self, ctx, evt_name, data1, data2):
+        assert ctx == self._dc_context
+        if hasattr(self, "_evlogger"):
+            self._evlogger(evt_name, data1, data2)
+            method = getattr(self, "on_" + evt_name.lower(), None)
+            if method is not None:
+                method(data1, data2)
+        return 0
+
+    def on_dc_event_imex_progress(self, data1, data2):
+        if data1 == 1000:
+            self._imex_events.put(True)
+        elif data1 == 0:
+            self._imex_events.put(False)
+
+    def on_dc_event_imex_file_written(self, data1, data2):
+        self._imex_events.put(data1)
+
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
@@ -543,122 +532,149 @@ class Account(object):
         if dc_res == 0:
             raise ValueError("no chat is streaming locations")
 
-    #
-    # meta API for start/stop and event based processing
-    #
 
-    def add_account_plugin(self, plugin, name=None):
-        """ add an account plugin which implements one or more of
-        the :class:`deltachat.hookspec.PerAccount` hooks.
-        """
-        self._pm.register(plugin, name=name)
-        self._pm.check_pending()
-        return plugin
+class IOThreads:
+    def __init__(self, dc_context, log_event=lambda *args: None):
+        self._dc_context = dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+        self._log_event = log_event
 
-    @contextmanager
-    def temp_plugin(self, plugin):
-        """ run a with-block with the given plugin temporarily registered. """
-        self._pm.register(plugin)
-        yield plugin
-        self._pm.unregister(plugin)
+    def is_started(self):
+        return len(self._name2thread) > 0
 
-    def stop_ongoing(self):
-        """ Stop ongoing securejoin, configuration or other core jobs. """
-        lib.dc_stop_ongoing_process(self._dc_context)
+    def start(self, imap=True, smtp=True, mvbox=False, sentbox=False):
+        assert not self.is_started()
+        if imap:
+            self._start_one_thread("inbox", self.imap_thread_run)
+        if mvbox:
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+        if sentbox:
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+        if smtp:
+            self._start_one_thread("smtp", self.smtp_thread_run)
 
-    def start(self, callback_thread=True):
-        """ start this account (activate imap/smtp threads etc.)
-        and return immediately.
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
 
-        If this account is not configured, an internal configuration
-        job will be scheduled if config values are sufficiently specified.
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        lib.dc_interrupt_mvbox_idle(self._dc_context)
+        lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                thread.join()
 
-        You may call `wait_shutdown` or `shutdown` after the
-        account is in started mode.
+    def imap_thread_run(self):
+        self._log_event("py-bindings-info", 0, "INBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_imap_jobs(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_fetch(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
 
-        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
+    def mvbox_thread_run(self):
+        self._log_event("py-bindings-info", 0, "MVBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_mvbox_jobs(self._dc_context)
+            lib.dc_perform_mvbox_fetch(self._dc_context)
+            lib.dc_perform_mvbox_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "MVBOX THREAD FINISHED")
 
-        :returns: None
-        """
-        if not self.is_configured():
-            if not self.get_config("addr") or not self.get_config("mail_pw"):
-                raise MissingCredentials("addr or mail_pwd not set in config")
-            lib.dc_configure(self._dc_context)
-        self._threads.start(callback_thread=callback_thread)
+    def sentbox_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SENTBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_sentbox_jobs(self._dc_context)
+            lib.dc_perform_sentbox_fetch(self._dc_context)
+            lib.dc_perform_sentbox_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SENTBOX THREAD FINISHED")
 
-    def wait_shutdown(self):
-        """ wait until shutdown of this account has completed. """
-        self._shutdown_event.wait()
+    def smtp_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_smtp_jobs(self._dc_context)
+            lib.dc_perform_smtp_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
 
-    def shutdown(self, wait=True):
-        """ shutdown account, stop threads and close and remove
-        underlying dc_context and callbacks. """
-        dc_context = self._dc_context
-        if dc_context is None:
-            return
 
-        if self._threads.is_started():
-            self.stop_ongoing()
-            self._threads.stop(wait=False)
-        lib.dc_close(dc_context)
-        self._hook_event_queue.put(None)
-        self._threads.stop(wait=wait)  # to wait for threads
-        self._dc_context = None
-        atexit.unregister(self.shutdown)
-        self._shutdown_event.set()
-        hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
+class EventLogger:
+    _loglock = threading.RLock()
 
-    def _handle_current_events(self):
-        """ handle all currently queued events and then return. """
+    def __init__(self, dc_context, logid=None, debug=True):
+        self._dc_context = dc_context
+        self._event_queue = Queue()
+        self._debug = debug
+        if logid is None:
+            logid = str(self._dc_context).strip(">").split()[-1]
+        self.logid = logid
+        self._timeout = None
+        self.init_time = time.time()
+
+    def __call__(self, evt_name, data1, data2):
+        self._log_event(evt_name, data1, data2)
+        self._event_queue.put((evt_name, data1, data2))
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get()
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout or self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev[0] == "DC_EVENT_ERROR":
+            raise ValueError("{}({!r},{!r})".format(*ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
             try:
-                event = self._hook_event_queue.get(block=False)
-            except queue.Empty:
+                ev = self._event_queue.get(False)
+            except Empty:
                 break
             else:
-                event.call_hook()
+                assert not rex.match(ev[0]), "event found {}".format(ev)
 
-    def iter_events(self, timeout=None):
-        """ yield hook events until shutdown.
-
-        It is not allowed to call iter_events() from multiple threads.
-        """
-        if self._in_use_iter_events:
-            raise RuntimeError("can only call iter_events() from one thread")
-        self._in_use_iter_events = True
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
-            event = self._hook_event_queue.get(timeout=timeout)
-            if event is None:
-                break
-            yield event
+            ev = self.get(timeout=timeout, check_error=check_error)
+            if rex.match(ev[0]):
+                return ev
 
-    def _map_ffi_event(self, ffi_event):
-        name = ffi_event.name
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-            data1 = ffi_event.data1
-            if data1 == 0 or data1 == 1000:
-                success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = self.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = self.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = self.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_CHAT_MODIFIED":
-            chat = self.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev[2]):
+                return ev
+
+    def _log_event(self, evt_name, data1, data2):
+        # don't show events that are anyway empty impls now
+        if evt_name == "DC_EVENT_GET_STRING":
+            return
+        if self._debug:
+            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
+            self._log(evpart)
+
+    def _log(self, msg):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        with self._loglock:
+            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
@@ -685,17 +701,3 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
-
-
-class HookEvent:
-    def __init__(self, account, name, kwargs):
-        assert hasattr(account._pm.hook, name), name
-        self.account = account
-        self.name = name
-        self.kwargs = kwargs
-
-    def call_hook(self):
-        hook = getattr(self.account._pm.hook, self.name, None)
-        if hook is None:
-            raise ValueError("event_name {} unknown".format(self.name))
-        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -30,7 +30,7 @@ class Chat(object):
         return not (self == other)
 
     def __repr__(self):
-        return "<Chat id={} name={}>".format(self.id, self.get_name())
+        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
 
     @property
     def _dc_chat(self):
@@ -51,16 +51,6 @@ class Chat(object):
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self):
-        """ return true if this chat is a group chat.
-
-        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
-        """
-        return lib.dc_chat_get_type(self._dc_chat) in (
-            const.DC_CHAT_TYPE_GROUP,
-            const.DC_CHAT_TYPE_VERIFIED_GROUP
-        )
-
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
 
@@ -68,13 +58,6 @@ class Chat(object):
         """
         return self.id == const.DC_CHAT_ID_DEADDROP
 
-    def is_muted(self):
-        """ return true if this chat is muted.
-
-        :returns: True if chat is muted, False otherwise.
-        """
-        return lib.dc_chat_is_muted(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -101,45 +84,14 @@ class Chat(object):
     def set_name(self, name):
         """ set name of this chat.
 
-        :param name: as a unicode string.
+        :param: name as a unicode string.
         :returns: None
         """
         name = as_dc_charpointer(name)
         return lib.dc_set_chat_name(self._dc_context, self.id, name)
 
-    def mute(self, duration=None):
-        """ mutes the chat
-
-        :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
-        :returns: None
-        """
-        if duration is None:
-            mute_duration = -1
-        else:
-            mute_duration = duration
-        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, mute_duration)
-        if not bool(ret):
-            raise ValueError("Call to dc_set_chat_mute_duration failed")
-
-    def unmute(self):
-        """ unmutes the chat
-
-        :returns: None
-        """
-        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, 0)
-        if not bool(ret):
-            raise ValueError("Failed to unmute chat")
-
-    def get_mute_duration(self):
-        """ Returns the number of seconds until the mute of this chat is lifted.
-
-        :param duration:
-        :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
-        """
-        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
-
     def get_type(self):
-        """ (deprecated) return type of this chat.
+        """ return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -157,30 +109,6 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg):
-        """send a message by using a ready Message object.
-
-        :param msg: a :class:`deltachat.message.Message` instance
-           previously returned by
-           e.g. :meth:`deltachat.message.Message.new_empty` or
-           :meth:`prepare_file`.
-        :raises ValueError: if message can not be sent.
-
-        :returns: a :class:`deltachat.message.Message` instance as
-           sent out.  This is the same object as was passed in, which
-           has been modified with the new state of the core.
-        """
-        if msg.is_out_preparing():
-            assert msg.id != 0
-            # get a fresh copy of dc_msg, the core needs it
-            msg = Message.from_db(self.account, msg.id)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
-        return msg
-
     def send_text(self, text):
         """ send a text message and return the resulting Message instance.
 
@@ -202,12 +130,9 @@ class Chat(object):
         :raises ValueError: if message can not be send/chat does not exist.
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
-        msg = Message.new_empty(self.account, view_type="file")
-        msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        return Message.from_db(self.account, sent_id)
+        msg = self.prepare_message_file(path=path, mime_type=mime_type)
+        self.send_prepared(msg)
+        return msg
 
     def send_image(self, path):
         """ send an image message and return the resulting Message instance.
@@ -217,12 +142,9 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         mime_type = mimetypes.guess_type(path)[0]
-        msg = Message.new_empty(self.account, view_type="image")
-        msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        return Message.from_db(self.account, sent_id)
+        msg = self.prepare_message_file(path=path, mime_type=mime_type, view_type="image")
+        self.send_prepared(msg)
+        return msg
 
     def prepare_message(self, msg):
         """ create a new prepared message.
@@ -363,7 +285,7 @@ class Chat(object):
             lib.dc_array_unref
         )
         return list(iter_array(
-            dc_array, lambda id: Contact(self.account, id))
+            dc_array, lambda id: Contact(self._dc_context, id))
         )
 
     def set_profile_image(self, img_path):
@@ -415,6 +337,12 @@ class Chat(object):
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
+    def get_subtitle(self):
+        """return the subtitle of the chat
+        :returns: the subtitle
+        """
+        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
+
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -427,7 +355,7 @@ class Chat(object):
         """return True if this chat is archived.
         :returns: True if archived.
         """
-        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+        return lib.dc_chat_get_archived(self._dc_chat)
 
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.

python/src/deltachat/const.py

@@ -11,7 +11,6 @@ from os.path import join as joinpath
 DC_GCL_ARCHIVED_ONLY = 0x01
 DC_GCL_NO_SPECIALS = 0x02
 DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
 DC_GCL_VERIFIED_ONLY = 0x01
 DC_GCL_ADD_SELF = 0x02
 DC_QR_ASK_VERIFYCONTACT = 200
@@ -19,7 +18,6 @@ DC_QR_ASK_VERIFYGROUP = 202
 DC_QR_FPR_OK = 210
 DC_QR_FPR_MISMATCH = 220
 DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
 DC_QR_ADDR = 320
 DC_QR_TEXT = 330
 DC_QR_URL = 332
@@ -70,6 +68,7 @@ DC_LP_SMTP_SOCKET_SSL = 0x20000
 DC_LP_SMTP_SOCKET_PLAIN = 0x40000
 DC_CERTCK_AUTO = 0
 DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_HOSTNAMES = 2
 DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
 DC_EMPTY_MVBOX = 0x01
 DC_EMPTY_INBOX = 0x02
@@ -99,22 +98,19 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091
 DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
 DC_PROVIDER_STATUS_OK = 1
 DC_PROVIDER_STATUS_PREPARATION = 2
 DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
 DC_STR_NOMESSAGES = 1
 DC_STR_SELF = 2
 DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
 DC_STR_VOICEMESSAGE = 7
 DC_STR_DEADDROP = 8
 DC_STR_IMAGE = 9
@@ -146,6 +142,7 @@ DC_STR_ARCHIVEDCHATS = 40
 DC_STR_STARREDMSGS = 41
 DC_STR_AC_SETUP_MSG_SUBJECT = 42
 DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
 DC_STR_CANNOT_LOGIN = 60
 DC_STR_SERVER_RESPONSE = 61
 DC_STR_MSGACTIONBYUSER = 62
@@ -161,7 +158,7 @@ DC_STR_COUNT = 68
 
 def read_event_defines(f):
     rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/contact.py

@@ -10,9 +10,8 @@ class Contact(object):
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
-    def __init__(self, account, id):
-        self.account = account
-        self._dc_context = account._dc_context
+    def __init__(self, dc_context, id):
+        self._dc_context = dc_context
         self.id = id
 
     def __eq__(self, other):
@@ -48,17 +47,3 @@ class Contact(object):
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
-
-    def get_profile_image(self):
-        """Get contact profile image.
-
-        :returns: path to profile image, None if no profile image exists.
-        """
-        dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        if dc_res == ffi.NULL:
-            return None
-        return from_dc_charpointer(dc_res)
-
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)

python/src/deltachat/eventlogger.py

@@ -1,137 +0,0 @@
-import deltachat
-import threading
-import time
-import re
-from queue import Queue, Empty
-from .hookspec import account_hookimpl, global_hookimpl
-
-
-@global_hookimpl
-def dc_account_init(account):
-    # send all FFI events for this account to a plugin hook
-    def _ll_event(ctx, evt_name, data1, data2):
-        assert ctx == account._dc_context
-        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-        account._pm.hook.ac_process_ffi_event(
-            account=account, ffi_event=ffi_event
-        )
-    deltachat.set_context_callback(account._dc_context, _ll_event)
-
-
-@global_hookimpl
-def dc_account_after_shutdown(dc_context):
-    deltachat.clear_context_callback(dc_context)
-
-
-class FFIEvent:
-    def __init__(self, name, data1, data2):
-        self.name = name
-        self.data1 = data1
-        self.data2 = data2
-
-    def __str__(self):
-        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
-
-
-class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
-    you'll get all ffi-events printed.
-    """
-    # to prevent garbled logging
-    _loglock = threading.RLock()
-
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
-        self.account = account
-        self.logid = logid
-        self.init_time = time.time()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._log_event(ffi_event)
-
-    def _log_event(self, ffi_event):
-        # don't show events that are anyway empty impls now
-        if ffi_event.name == "DC_EVENT_GET_STRING":
-            return
-        self.account.ac_log_line(str(ffi_event))
-
-    @account_hookimpl
-    def ac_log_line(self, message):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        elapsed = time.time() - self.init_time
-        locname = tname
-        if self.logid:
-            locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-        with self._loglock:
-            print(s, flush=True)
-
-
-class FFIEventTracker:
-    def __init__(self, account, timeout=None):
-        self.account = account
-        self._timeout = timeout
-        self._event_queue = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._event_queue.put(ffi_event)
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get(check_error=check_error)
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout if timeout is not None else self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError(str(ev))
-        return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev.name):
-                return ev
-
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
-                return ev
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.account.get_message_by_id(ev.data2)
-
-    def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
-        if the event contains no msgid"""
-        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-        if ev.data2 > 0:
-            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/hookspec.py

@@ -1,92 +0,0 @@
-""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
-
-import pluggy
-
-
-account_spec_name = "deltachat-account"
-account_hookspec = pluggy.HookspecMarker(account_spec_name)
-account_hookimpl = pluggy.HookimplMarker(account_spec_name)
-
-global_spec_name = "deltachat-global"
-global_hookspec = pluggy.HookspecMarker(global_spec_name)
-global_hookimpl = pluggy.HookimplMarker(global_spec_name)
-
-
-class PerAccount:
-    """ per-Account-instance hook specifications.
-
-    Except for ac_process_ffi_event all hooks are executed
-    in the thread which calls Account.wait_shutdown().
-    """
-    @classmethod
-    def _make_plugin_manager(cls):
-        pm = pluggy.PluginManager(account_spec_name)
-        pm.add_hookspecs(cls)
-        return pm
-
-    @account_hookspec
-    def ac_process_ffi_event(self, ffi_event):
-        """ process a CFFI low level events for a given account.
-
-        ffi_event has "name", "data1", "data2" values as specified
-        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
-        """
-
-    @account_hookspec
-    def ac_log_line(self, message):
-        """ log a message related to the account. """
-
-    @account_hookspec
-    def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
-
-    @account_hookspec
-    def ac_incoming_message(self, message):
-        """ Called on any incoming message (to deaddrop or chat). """
-
-    @account_hookspec
-    def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
-
-    @account_hookspec
-    def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
-
-    @account_hookspec
-    def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
-
-    @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
-
-    @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
-
-
-class Global:
-    """ global hook specifications using a per-process singleton
-    plugin manager instance.
-
-    """
-    _plugin_manager = None
-
-    @classmethod
-    def _get_plugin_manager(cls):
-        if cls._plugin_manager is None:
-            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
-            pm.add_hookspecs(cls)
-        return cls._plugin_manager
-
-    @global_hookspec
-    def dc_account_init(self, account):
-        """ called when `Account::__init__()` function starts executing. """
-
-    @global_hookspec
-    def dc_account_after_shutdown(self, account, dc_context):
-        """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -1,106 +0,0 @@
-
-import threading
-import time
-
-from contextlib import contextmanager
-
-from .capi import lib
-
-
-class IOThreads:
-    def __init__(self, account):
-        self.account = account
-        self._dc_context = account._dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-
-    def is_started(self):
-        return len(self._name2thread) > 0
-
-    def start(self, callback_thread):
-        assert not self.is_started()
-        self._start_one_thread("inbox", self.imap_thread_run)
-        self._start_one_thread("smtp", self.smtp_thread_run)
-
-        if callback_thread:
-            self._start_one_thread("cb", self.cb_thread_run)
-
-        if int(self.account.get_config("mvbox_watch")):
-            self._start_one_thread("mvbox", self.mvbox_thread_run)
-
-        if int(self.account.get_config("sentbox_watch")):
-            self._start_one_thread("sentbox", self.sentbox_thread_run)
-
-    def _start_one_thread(self, name, func):
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        t.setDaemon(1)
-        t.start()
-
-    @contextmanager
-    def log_execution(self, message):
-        self.account.ac_log_line(message + " START")
-        yield
-        self.account.ac_log_line(message + " FINISHED")
-
-    def stop(self, wait=False):
-        self._thread_quitflag = True
-
-        # Workaround for a race condition. Make sure that thread is
-        # not in between checking for quitflag and entering idle.
-        time.sleep(0.5)
-
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        if "mvbox" in self._name2thread:
-            lib.dc_interrupt_mvbox_idle(self._dc_context)
-        if "sentbox" in self._name2thread:
-            lib.dc_interrupt_sentbox_idle(self._dc_context)
-        if wait:
-            for name, thread in self._name2thread.items():
-                if thread != threading.currentThread():
-                    thread.join()
-
-    def cb_thread_run(self):
-        with self.log_execution("CALLBACK THREAD START"):
-            it = self.account.iter_events()
-            while not self._thread_quitflag:
-                try:
-                    ev = next(it)
-                except StopIteration:
-                    break
-                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
-                ev.call_hook()
-
-    def imap_thread_run(self):
-        with self.log_execution("INBOX THREAD START"):
-            while not self._thread_quitflag:
-                lib.dc_perform_imap_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_idle(self._dc_context)
-
-    def mvbox_thread_run(self):
-        with self.log_execution("MVBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_mvbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_idle(self._dc_context)
-
-    def sentbox_thread_run(self):
-        with self.log_execution("SENTBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_sentbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_idle(self._dc_context)
-
-    def smtp_thread_run(self):
-        with self.log_execution("SMTP THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_smtp_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -28,9 +28,7 @@ class Message(object):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        c = self.get_sender_contact()
-        return "<Message id={} sender={}/{} outgoing={} chat={}/{}>".format(
-            self.id, c.id, c.addr, self.is_outgoing(), self.chat.id, self.chat.get_name())
+        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
 
     @classmethod
     def from_db(cls, account, id):
@@ -52,16 +50,6 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
-        """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self._dc_context, self.id),
-                lib.dc_msg_unref
-        )
-
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -93,10 +81,6 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
-    def is_system_message(self):
-        """ return True if this message is a system/info message. """
-        return lib.dc_msg_is_info(self._dc_msg)
-
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
@@ -175,13 +159,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    def get_sender_chat(self):
-        """return the 1:1 chat with the sender of this message.
-
-        :returns: :class:`deltachat.chat.Chat` instance
-        """
-        return self.get_sender_contact().get_chat()
-
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
@@ -189,7 +166,7 @@ class Message(object):
         """
         from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self.account, contact_id)
+        return Contact(self._dc_context, contact_id)
 
     #
     # Message State query methods
@@ -197,7 +174,7 @@ class Message(object):
     @property
     def _msgstate(self):
         if self.id == 0:
-            dc_msg = self._dc_msg
+            dc_msg = self.message._dc_msg
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
@@ -230,13 +207,6 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
-    def is_outgoing(self):
-        """Return True if Message is outgoing. """
-        return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
-
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -299,10 +269,6 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
-    def mark_seen(self):
-        """ mark this message as seen. """
-        self.account.mark_seen_messages([self.id])
-
 
 # some code for handling DC_MSG_* view types
 
@@ -322,29 +288,3 @@ def get_viewtype_code_from_name(view_type_name):
             return code
     raise ValueError("message typecode not found for {!r}, "
                      "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
-
-
-#
-# some helper code for turning system messages into hook events
-#
-
-def map_system_message(msg):
-    if msg.is_system_message():
-        res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
-
-
-def parse_system_add_remove(text):
-    # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
-    text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")

python/src/deltachat/provider.py

@@ -11,18 +11,27 @@ class ProviderNotFoundError(Exception):
 class Provider(object):
     """Provider information.
 
-    :param domain: The email to get the provider info for.
+    :param domain: The domain to get the provider info for, this is
+    normally the part following the `@` of the domain.
     """
 
-    def __init__(self, account, addr):
+    def __init__(self, domain):
         provider = ffi.gc(
-            lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
+            lib.dc_provider_new_from_domain(as_dc_charpointer(domain)),
             lib.dc_provider_unref,
         )
         if provider == ffi.NULL:
             raise ProviderNotFoundError("Provider not found")
         self._provider = provider
 
+    @classmethod
+    def from_email(cls, email):
+        """Create provider info from an email address.
+
+        :param email: Email address to get provider info for.
+        """
+        return cls(email.split('@')[-1])
+
     @property
     def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
@@ -30,10 +39,21 @@ class Provider(object):
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self):
-        """Should be shown to the user on login."""
+    def name(self):
+        """The name of the provider."""
+        return from_dc_charpointer(lib.dc_provider_get_name(self._provider))
+
+    @property
+    def markdown(self):
+        """Content of the information page, formatted as markdown."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hints(self._provider))
+            lib.dc_provider_get_markdown(self._provider))
+
+    @property
+    def status_date(self):
+        """The date the provider info was last updated, as a string."""
+        return from_dc_charpointer(
+            lib.dc_provider_get_status_date(self._provider))
 
     @property
     def status(self):

python/src/deltachat/testplugin.py

@@ -1,439 +0,0 @@
-from __future__ import print_function
-import os
-import sys
-import subprocess
-import queue
-import threading
-import fnmatch
-import time
-import weakref
-import tempfile
-
-import pytest
-import requests
-
-from . import Account, const
-from .tracker import ConfigureTracker
-from .capi import lib
-from .eventlogger import FFIEventLogger, FFIEventTracker
-from _pytest.monkeypatch import MonkeyPatch
-from _pytest._code import Source
-
-import deltachat
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-    parser.addoption(
-        "--ignored", action="store_true",
-        help="Also run tests marked with the ignored marker",
-    )
-
-
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
-        if cfg:
-            config.option.liveconfig = cfg
-
-    # Make sure we don't get garbled output because threads keep running
-    # collect all ever created accounts in a weakref-set (so we don't
-    # keep objects unneccessarily alive) and enable/disable logging
-    # for each pytest test phase # (setup/call/teardown).
-    # Additionally make the acfactory use a logging/no-logging default.
-
-    class LoggingAspect:
-        def __init__(self):
-            self._accounts = weakref.WeakSet()
-
-        @deltachat.global_hookimpl
-        def dc_account_init(self, account):
-            self._accounts.add(account)
-
-        def disable_logging(self, item):
-            for acc in self._accounts:
-                acc.disable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(False)
-
-        def enable_logging(self, item):
-            for acc in self._accounts:
-                acc.enable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(True)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_setup(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_pyfunc_call(self, pyfuncitem):
-            self.enable_logging(pyfuncitem)
-            yield
-            self.disable_logging(pyfuncitem)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_teardown(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-    la = LoggingAspect()
-    config.pluginmanager.register(la)
-    deltachat.register_global_plugin(la)
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    m = MonkeyPatch()
-    try:
-        m.setattr(sys.stdout, "write", lambda x: len(x))
-        ac = Account(t)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        m.undo()
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "?" in cfg:
-            url, token = cfg.split("?", 1)
-            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url):
-        self.configlist = []
-        self.url = url
-
-    def get(self, index):
-        try:
-            return self.configlist[index]
-        except IndexError:
-            assert index == len(self.configlist), index
-            res = requests.post(self.url)
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-            return config
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            return SessionLiveConfigFromURL(liveconfig_opt)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture
-def data(request):
-    class Data:
-        def __init__(self):
-            # trying to find test data heuristically
-            # because we are run from a dev-setup with pytest direct,
-            # through tox, and then maybe also from deltachat-binding
-            # users like "deltabot".
-            self.paths = [os.path.normpath(x) for x in [
-                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
-                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
-            ]]
-
-        def get_path(self, bn):
-            """ return path of file or None if it doesn't exist. """
-            for path in self.paths:
-                fn = os.path.join(path, *bn.split("/"))
-                if os.path.exists(fn):
-                    return fn
-            print("WARNING: path does not exist: {!r}".format(fn))
-
-        def read_path(self, bn, mode="r"):
-            fn = self.get_path(bn)
-            if fn is not None:
-                with open(fn, mode) as f:
-                    return f.read()
-
-    return Data()
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self._accounts = []
-            self.init_time = time.time()
-            self._generated_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
-            self.set_logging_default(False)
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-            while self._accounts:
-                acc = self._accounts.pop()
-                acc.shutdown()
-                acc.disable_logging()
-
-        def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
-            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
-            if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
-            self._accounts.append(ac)
-            return ac
-
-        def set_logging_default(self, logging):
-            self._logging = bool(logging)
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
-
-        def _preconfigure_key(self, account, addr):
-            # Only set a key if we haven't used it yet for another account.
-            if self._generated_keys:
-                keyname = self._generated_keys.pop(0)
-                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
-                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
-                if fname_pub and fname_sec:
-                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                    return True
-                else:
-                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            self._preconfigure_key(ac, addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def get_online_config(self, pre_generated_key=True, quiet=False):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
-            if pre_generated_key:
-                self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
-            return ac, dict(configdict)
-
-        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
-                                           pre_generated_key=True, quiet=False, config={}):
-            ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key, quiet=quiet)
-            configdict.update(config)
-            configdict["mvbox_watch"] = str(int(mvbox))
-            configdict["mvbox_move"] = str(int(move))
-            configdict["sentbox_watch"] = str(int(sentbox))
-            ac.update_config(configdict)
-            ac.start()
-            return ac
-
-        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
-            ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1._configtracker.wait_imap_connected()
-            ac1._configtracker.wait_smtp_connected()
-            ac1._configtracker.wait_finish()
-            return ac1
-
-        def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
-            ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1._configtracker.wait_finish()
-            ac2._configtracker.wait_finish()
-            return ac1, ac2
-
-        def clone_online_account(self, account, pre_generated_key=True):
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
-            ac.update_config(dict(
-                addr=account.get_config("addr"),
-                mail_pw=account.get_config("mail_pw"),
-                mvbox_watch=account.get_config("mvbox_watch"),
-                mvbox_move=account.get_config("mvbox_move"),
-                sentbox_watch=account.get_config("sentbox_watch"),
-            ))
-            ac.start()
-            return ac
-
-        def run_bot_process(self, module, ffi=True):
-            fn = module.__file__
-
-            bot_ac, bot_cfg = self.get_online_config()
-
-            args = [
-                sys.executable,
-                "-u",
-                fn,
-                "--email", bot_cfg["addr"],
-                "--password", bot_cfg["mail_pw"],
-                bot_ac.db_path,
-            ]
-            if ffi:
-                args.insert(-1, "--show-ffi")
-            print("$", " ".join(args))
-            popen = subprocess.Popen(
-                args=args,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=0,                 # line buffering
-                close_fds=True,            # close all FDs other than 0/1/2
-                universal_newlines=True    # give back text
-            )
-            bot = BotProcess(popen, bot_cfg)
-            self._finalizers.append(bot.kill)
-            return bot
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
-    return am
-
-
-class BotProcess:
-    def __init__(self, popen, bot_cfg):
-        self.popen = popen
-        self.addr = bot_cfg["addr"]
-
-        # we read stdout as quickly as we can in a thread and make
-        # the (unicode) lines available for readers through a queue.
-        self.stdout_queue = queue.Queue()
-        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(1)
-        t.start()
-
-    def _run_stdout_thread(self):
-        try:
-            while 1:
-                line = self.popen.stdout.readline()
-                if not line:
-                    break
-                line = line.strip()
-                self.stdout_queue.put(line)
-        finally:
-            self.stdout_queue.put(None)
-
-    def kill(self):
-        self.popen.kill()
-
-    def wait(self, timeout=30):
-        self.popen.wait(timeout=timeout)
-
-    def fnmatch_lines(self, pattern_lines):
-        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
-        for next_pattern in patterns:
-            print("+++FNMATCH:", next_pattern)
-            ignored = []
-            while 1:
-                line = self.stdout_queue.get(timeout=15)
-                if line is None:
-                    if ignored:
-                        print("BOT stdout terminated after these lines")
-                        for line in ignored:
-                            print(line)
-                    raise IOError("BOT stdout-thread terminated")
-                if fnmatch.fnmatch(line, next_pattern):
-                    print("+++MATCHED:", line)
-                    break
-                else:
-                    print("+++IGN:", line)
-                    ignored.append(line)
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-    return Printer()

python/src/deltachat/tracker.py

@@ -1,76 +0,0 @@
-
-from queue import Queue
-from threading import Event
-
-from .hookspec import account_hookimpl
-
-
-class ImexFailed(RuntimeError):
-    """ Exception for signalling that import/export operations failed."""
-
-
-class ImexTracker:
-    def __init__(self):
-        self._imex_events = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
-            self._imex_events.put(ffi_event.data1)
-        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(ffi_event.data1)
-
-    def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
-        files_written = []
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif ev == 0:
-                raise ImexFailed("export failed, exp-files: {}".format(files_written))
-            elif ev == 1000:
-                return files_written
-
-
-class ConfigureFailed(RuntimeError):
-    """ Exception for signalling that configuration failed."""
-
-
-class ConfigureTracker:
-    ConfigureFailed = ConfigureFailed
-
-    def __init__(self):
-        self._configure_events = Queue()
-        self._smtp_finished = Event()
-        self._imap_finished = Event()
-        self._ffi_events = []
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._ffi_events.append(ffi_event)
-        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
-            self._smtp_finished.set()
-        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
-            self._imap_finished.set()
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        self._configure_events.put(success)
-
-    def wait_smtp_connected(self):
-        """ wait until smtp is configured. """
-        self._smtp_finished.wait()
-
-    def wait_imap_connected(self):
-        """ wait until smtp is configured. """
-        self._imap_finished.wait()
-
-    def wait_finish(self):
-        """ wait until configure is completed.
-
-        Raise Exception if Configure failed
-        """
-        if not self._configure_events.get():
-            content = "\n".join(map(str, self._ffi_events))
-            raise ConfigureFailed(content)

python/tests/conftest.py

@@ -1,18 +1,268 @@
 from __future__ import print_function
+import os
+import pytest
+import requests
+import time
+from deltachat import Account
+from deltachat import const
+from deltachat.capi import lib
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+
+
+def pytest_configure(config):
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_PY_LIVECONFIG')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    try:
+        ac = Account(t, eventlogging=False)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "#" in cfg:
+            url, token = cfg.split("#", 1)
+            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+@pytest.fixture(scope="session")
+def data():
+    class Data:
+        def __init__(self):
+            self.path = os.path.join(os.path.dirname(__file__), "data")
+
+        def get_path(self, bn):
+            fn = os.path.join(self.path, bn)
+            assert os.path.exists(fn)
+            return fn
+    return Data()
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url, create_token):
+        self.configlist = []
+        for i in range(2):
+            res = requests.post(url, json={"token_create_user": int(create_token)})
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            url, create_token = liveconfig_opt.split("#", 1)
+            return SessionLiveConfigFromURL(url, create_token)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid):
+            ac = Account(path, logid=logid)
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(2)
+            return ac
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def peek_online_config(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            return session_liveconfig.get(self.live_count)
+
+        def get_online_config(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False):
+            ac, configdict = self.get_online_config()
+            ac.configure(**configdict)
+            ac.start_threads(mvbox=mvbox, sentbox=sentbox)
+            return ac
+
+        def get_one_online_account(self):
+            ac1 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            return ac1
+
+        def get_two_online_accounts(self):
+            ac1 = self.get_online_configuring_account()
+            ac2 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            wait_successful_IMAP_SMTP_connection(ac2)
+            wait_configuration_progress(ac2, 1000)
+            return ac1, ac2
+
+        def clone_online_account(self, account):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
+            ac.start_threads()
+            return ac
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()
 
 
 def wait_configuration_progress(account, min_target, max_target=1001):
     min_target = min(min_target, max_target)
     while 1:
-        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        if event.data1 >= min_target and event.data1 <= max_target:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if data1 >= min_target and data1 <= max_target:
             print("** CONFIG PROGRESS {}".format(min_target), account)
             break
 
 
 def wait_securejoin_inviter_progress(account, target):
     while 1:
-        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-        if event.data2 >= target:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if data2 >= target:
             print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
             break
+
+
+def wait_successful_IMAP_SMTP_connection(account):
+    imap_ok = smtp_ok = False
+    while not imap_ok or not smtp_ok:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
+        if evt_name == "DC_EVENT_IMAP_CONNECTED":
+            imap_ok = True
+            print("** IMAP OK", account)
+        if evt_name == "DC_EVENT_SMTP_CONNECTED":
+            smtp_ok = True
+            print("** SMTP OK", account)
+    print("** IMAP and SMTP logins successful", account)
+
+
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev[1] == chat_id
+    if msg_id is not None:
+        assert ev[2] == msg_id
+    return ev[2]

python/tests/data/key

@@ -1 +0,0 @@
-../../../test-data/key

python/tests/test_increation.py

@@ -1,57 +1,10 @@
 from __future__ import print_function
-
-import os.path
-import shutil
-
-import pytest
 from filecmp import cmp
-
-from conftest import wait_configuration_progress
 from deltachat import const
-
-
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev.data1 == chat_id
-    if msg_id is not None:
-        assert ev.data2 == msg_id
-    return ev.data2
+from conftest import wait_configuration_progress, wait_msgs_changed
 
 
 class TestOnlineInCreation:
-    def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-
-        lp.sec("Creating in-creation file outside of blobdir")
-        assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt').ensure(file=1)
-        with pytest.raises(Exception):
-            chat.prepare_message_file(src.strpath)
-
-    def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-
-        lp.sec("Creating file outside of blobdir")
-        assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt')
-        src.write("hello there\n")
-        chat.send_file(src.strpath)
-
-        blob_src = os.path.join(ac1.get_blobdir(), 'file.txt')
-        assert os.path.exists(blob_src), "file.txt not copied to blobdir"
-
     def test_forward_increation(self, acfactory, data, lp):
         ac1 = acfactory.get_online_configuring_account()
         ac2 = acfactory.get_online_configuring_account()
@@ -64,10 +17,7 @@ class TestOnlineInCreation:
         wait_msgs_changed(ac1, 0, 0)  # why no chat id?
 
         lp.sec("create a message with a file in creation")
-        orig = data.get_path("d.png")
-        path = os.path.join(ac1.get_blobdir(), 'd.png')
-        with open(path, "x") as fp:
-            fp.write("preparing")
+        path = data.get_path("d.png")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -88,7 +38,6 @@ class TestOnlineInCreation:
 
         lp.sec("finish creating the file and send it")
         assert prepared_original.is_out_preparing()
-        shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -99,22 +48,22 @@ class TestOnlineInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat.id
-        assert ev.data2 == prepared_original.id
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat2.id
-        assert ev.data2 == forwarded_id
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev[1] == chat.id
+        assert ev[2] == prepared_original.id
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev[1] == chat2.id
+        assert ev[2] == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1.data2)
-        assert cmp(received_original.filename, orig, shallow=False)
+        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1[2])
+        assert cmp(received_original.filename, path, False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2.data1 != ev1.data1
-        received_copy = ac2.get_message_by_id(ev2.data2)
-        assert cmp(received_copy.filename, orig, shallow=False)
+        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2[1] != ev1[1]
+        received_copy = ac2.get_message_by_id(ev2[2])
+        assert cmp(received_copy.filename, path, False)

python/tests/test_lowlevel.py

@@ -1,9 +1,8 @@
 from __future__ import print_function
 from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
-from deltachat import register_global_plugin
-from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
+from deltachat.account import EventLogger
 
 
 def test_empty_context():
@@ -18,31 +17,29 @@ def test_callback_None2int():
     clear_context_callback(ctx)
 
 
-def test_dc_close_events(tmpdir, acfactory):
-    ac1 = acfactory.get_unconfigured_account()
-
-    # register after_shutdown function
-    shutdowns = []
-
-    class ShutdownPlugin:
-        @global_hookimpl
-        def dc_account_after_shutdown(self, account):
-            assert account._dc_context is None
-            shutdowns.append(account)
-    register_global_plugin(ShutdownPlugin())
-    assert hasattr(ac1, "_dc_context")
-    ac1.shutdown()
-    assert shutdowns == [ac1]
+def test_dc_close_events(tmpdir):
+    ctx = ffi.gc(
+        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    evlog = EventLogger(ctx)
+    evlog.set_timeout(5)
+    set_context_callback(
+        ctx,
+        lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
+    )
+    p = tmpdir.join("hello.db")
+    lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
+    capi.lib.dc_close(ctx)
 
     def find(info_string):
-        evlog = ac1._evtracker
         while 1:
             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
-            data2 = ev.data2
+            data2 = ev[2]
             if info_string in data2:
                 return
             else:
-                print("skipping event", ev)
+                print("skipping event", *ev)
 
     find("disconnecting inbox-thread")
     find("disconnecting sentbox-thread")
@@ -92,10 +89,10 @@ def test_markseen_invalid_message_ids(acfactory):
     contact1 = ac1.create_contact(email="some1@example.com", name="some1")
     chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
-    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
 def test_get_special_message_id_returns_empty_message(acfactory):
@@ -105,12 +102,19 @@ def test_get_special_message_id_returns_empty_message(acfactory):
         assert msg.id == 0
 
 
+def test_provider_info():
+    provider = lib.dc_provider_new_from_email(cutil.as_dc_charpointer("ex@example.com"))
+    assert cutil.from_dc_charpointer(
+        lib.dc_provider_get_overview_page(provider)
+    ) == "https://providers.delta.chat/example.com"
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_name(provider)) == "Example"
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_markdown(provider)) == "\n..."
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_status_date(provider)) == "2018-09"
+    assert lib.dc_provider_get_status(provider) == const.DC_PROVIDER_STATUS_PREPARATION
+
+
 def test_provider_info_none():
-    ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
+    assert lib.dc_provider_new_from_email(cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
 
 
 def test_get_info_closed():

python/tests/test_provider_info.py

@@ -0,0 +1,27 @@
+import pytest
+
+from deltachat import const
+from deltachat import provider
+
+
+def test_provider_info_from_email():
+    example = provider.Provider.from_email("email@example.com")
+    assert example.overview_page == "https://providers.delta.chat/example.com"
+    assert example.name == "Example"
+    assert example.markdown == "\n..."
+    assert example.status_date == "2018-09"
+    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
+
+
+def test_provider_info_from_domain():
+    example = provider.Provider("example.com")
+    assert example.overview_page == "https://providers.delta.chat/example.com"
+    assert example.name == "Example"
+    assert example.markdown == "\n..."
+    assert example.status_date == "2018-09"
+    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
+
+
+def test_provider_info_none():
+    with pytest.raises(provider.ProviderNotFoundError):
+        provider.Provider.from_email("email@unexistent.no")

python/tox.ini

@@ -7,14 +7,13 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx {posargs:tests} 
+    # python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
-    DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -41,13 +40,13 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/ examples/
+    flake8 tests/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
+    sphinx==2.2.0
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
@@ -66,10 +65,11 @@ commands =
 
 
 [pytest]
-addopts = -v -ra --strict-markers
+addopts = -v -rs 
+python_files = tests/test_*.py 
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 90
+timeout = 60 
 timeout_method = thread
 
 [flake8]

rust-toolchain

@@ -1 +1 @@
-nightly-2020-03-12
+nightly-2019-11-06

scripts/proxy.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Examples:
-#
-# Original server that doesn't use SSL:
-# ./proxy.py 8080 imap.nauta.cu 143
-# ./proxy.py 8081 smtp.nauta.cu 25
-#
-# Original server that uses SSL:
-# ./proxy.py 8080 testrun.org 993 --ssl
-# ./proxy.py 8081 testrun.org 465 --ssl
-
-from datetime import datetime
-import argparse
-import selectors
-import ssl
-import socket
-import socketserver
-
-
-class Proxy(socketserver.ThreadingTCPServer):
-    allow_reuse_address = True
-
-    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
-        self.real_host = real_host
-        self.real_port = real_port
-        self.use_ssl = use_ssl
-        super().__init__((proxy_host, proxy_port), RequestHandler)
-
-
-class RequestHandler(socketserver.BaseRequestHandler):
-
-    def handle(self):
-        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
-
-        total = 0
-        real_server = (self.server.real_host, self.server.real_port)
-        with socket.create_connection(real_server) as sock:
-            if self.server.use_ssl:
-                context = ssl.create_default_context()
-                sock = context.wrap_socket(
-                    sock, server_hostname=real_server[0])
-
-            forward = {self.request: sock, sock: self.request}
-
-            sel = selectors.DefaultSelector()
-            sel.register(self.request, selectors.EVENT_READ,
-                         self.client_address)
-            sel.register(sock, selectors.EVENT_READ, real_server)
-
-            active = True
-            while active:
-                events = sel.select()
-                for key, mask in events:
-                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
-                    data = key.fileobj.recv(1024)
-                    received = len(data)
-                    total += received
-                    print(data)
-                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
-                    if data:
-                        forward[key.fileobj].sendall(data)
-                    else:
-                        print('\nCLOSING CONNECTION.\n\n')
-                        forward[key.fileobj].close()
-                        key.fileobj.close()
-                        active = False
-
-
-if __name__ == '__main__':
-    p = argparse.ArgumentParser(description='Simple Python Proxy')
-    p.add_argument(
-        "proxy_port", help="the port where the proxy will listen", type=int)
-    p.add_argument('host', help="the real host")
-    p.add_argument('port', help="the port of the real host", type=int)
-    p.add_argument("--ssl", help="use ssl to connect to the real host",
-                   action="store_true")
-    args = p.parse_args()
-
-    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
-        proxy.serve_forever()

set_core_version.py

@@ -35,7 +35,7 @@ if __name__ == "__main__":
     if len(sys.argv) < 2:
         for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
             print("{}: {}".format(x, read_toml_version(x)))
-        raise SystemExit("need argument: new version, example: 1.25.0")
+        raise SystemExit("need argument: new version, example 1.0.0-beta.27")
     newversion = sys.argv[1]
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
@@ -45,7 +45,7 @@ if __name__ == "__main__":
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
     for line in open("CHANGELOG.md"):
-        ## 1.25.0
+        ## 1.0.0-beta5
         if line.startswith("## "):
             if line[2:].strip().startswith(newversion):
                 break
@@ -55,9 +55,7 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
-    subprocess.call(["cargo", "check"])
-    subprocess.call(["git", "add", "-u"])
-    # subprocess.call(["cargo", "update", "-p", "deltachat"])
+    subprocess.call(["cargo", "update", "-p", "deltachat"])
 
     print("after commit make sure to: ")
     print("")

spec.md

@@ -1,6 +1,6 @@
 # Chat-over-Email specification
 
-Version 0.30.0
+Version 0.19.0
 
 This document describes how emails can be used
 to implement typical messenger functions
@@ -17,9 +17,6 @@ while staying compatible to existing MUAs.
     - [Change group name](#change-group-name)
     - [Set group image](#set-group-image)
 - [Set profile image](#set-profile-image)
-- [Locations](#locations)
-    - [User locations](#user-locations)
-    - [Points of interest](#points-of-interest)
 - [Miscellaneous](#miscellaneous)
 
 
@@ -32,7 +29,8 @@ Messages SHOULD be encrypted by the
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
 by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
 If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+the subject of encrypted messages SHOULD be replaced by the string
+`Chat: Encrypted message` where the part after the colon MAY be localized.
 
 
 # Outgoing messages
@@ -115,7 +113,6 @@ but MUAs typically expose the sender in the UI.
 Groups are chats with usually more than one recipient,
 each defined by an email-address.
 The sender plus the recipients are the group members.
-All group members form the member list.
 
 To allow different groups with the same members,
 groups are identified by a group-id.
@@ -138,7 +135,8 @@ The group-name MUST be written to `Chat-Group-Name` header
 to join a group chat on a second device any time).
 
 The `Subject` header of outgoing group messages
-SHOULD be set to the group-name.
+SHOULD start with the characters `Chat:`
+followed by the group-name and a colon followed by an excerpt of the message.
 
 To identify the group-id on replies from normal MUAs,
 the group-id MUST also be added to the message-id of outgoing messages.
@@ -155,8 +153,8 @@ The message-id MUST have the format `Gr.<group-id>.<unique data>`.
     Hello group - this group contains three members
 
 Messengers adding the member list in the form `Name <email-address>`
-MUST take care only to distribute the names authorized by the contacts themselves.
-Otherwise, names as _Daddy_ or _Honey_ may be distributed
+MUST take care only to spread the names authorized by the contacts themselves.
+Otherwise, names as _Daddy_ or _Honey_ may be spread
 (this issue is also true for normal MUAs, however,
 for more contact- and chat-centralized apps
 such situations happen more frequently).
@@ -179,22 +177,12 @@ to a normal single-user chat with the email-address given in `From`.
 
 ## Add and remove members
 
-Messenger clients MUST init the member list
-from the `From`/`To` headers on the first group message.
-
-When a member is added later,
-a `Chat-Group-Member-Added` action header must be sent
-with the value set to the email-address of the added member.
-When receiving a `Chat-Group-Member-Added` header, however,
-_all missing_ members  the `From`/`To` headers has to be added.
-This is to mitigate problems when receiving messages
-in different orders, esp. on creating new groups.
-
-To remove a member, a `Chat-Group-Member-Removed` header must be sent
-with the value set to the email-address of the member to remove.
-When receiving a `Chat-Group-Member-Removed` header,
-only exaxtly the given member has to be removed from the member list.
-
+Messenger clients MUST construct the member list
+from the `From`/`To` headers only on the first group message
+or if they see a `Chat-Group-Member-Added`
+or `Chat-Group-Member-Removed` action header.
+Both headers MUST have the email-address
+of the added or removed member as the value.
 Messenger clients MUST NOT construct the member list
 on other group messages
 (this is to avoid accidentally altered To-lists in normal MUAs;
@@ -260,11 +248,11 @@ and the message SHOULD appear as a message or action from the sender.
 A group MAY have a group-image.
 To change or set the group-image,
 the messenger MUST attach an image file to a message
-and MUST add the header `Chat-Group-Avatar`
+and MUST add the header `Chat-Group-Image`
 with the value set to the image name.
 
 To remove the group-image,
-the messenger MUST add the header `Chat-Group-Avatar: 0`.
+the messenger MUST add the header `Chat-Group-Image: 0`.
 
 The messenger SHOULD send an explicit mail for each group image change.
 The body of the message SHOULD contain
@@ -277,7 +265,7 @@ and the message SHOULD appear as a message or action from the sender.
     Chat-Version: 1.0
     Chat-Group-ID: 12345uvwxyZ
     Chat-Group-Name: Our Group
-    Chat-Group-Avatar: image.jpg
+    Chat-Group-Image: image.jpg
     Message-ID: Gr.12345uvwxyZ.0005@domain
     Subject: Chat: Our Group: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
@@ -295,25 +283,25 @@ and the message SHOULD appear as a message or action from the sender.
 
 The image format SHOULD be image/jpeg or image/png.
 To save data, it is RECOMMENDED
-to add a `Chat-Group-Avatar` only on image changes.
+to add a `Chat-Group-Image` only on image changes.
 
 
 # Set profile image
 
-A user MAY have a profile-image that MAY be distributed to their contacts.
+A user MAY have a profile-image that MAY be spread to his contacts.
 To change or set the profile-image,
 the messenger MUST attach an image file to a message
-and MUST add the header `Chat-User-Avatar`
+and MUST add the header `Chat-Profile-Image`
 with the value set to the image name.
 
 To remove the profile-image,
-the messenger MUST add the header `Chat-User-Avatar: 0`.
+the messenger MUST add the header `Chat-Profile-Image: 0`.
 
-To distribute the image,
+To spread the image,
 the messenger MAY send the profile image
 together with the next mail to a given contact
 (to do this only once,
-the messenger has to keep a `user_avatar_update_state` somewhere).
+the messenger has to keep a `profile_image_update_state` somewhere).
 Alternatively, the messenger MAY send an explicit mail
 for each profile-image change to all contacts using a compatible messenger.
 The messenger SHOULD NOT send an explicit mail to normal MUAs.
@@ -321,7 +309,7 @@ The messenger SHOULD NOT send an explicit mail to normal MUAs.
     From: sender@domain
     To: rcpt@domain
     Chat-Version: 1.0
-    Chat-User-Avatar: photo.jpg
+    Chat-Profile-Image: photo.jpg
     Subject: Chat: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
 
@@ -337,71 +325,13 @@ The messenger SHOULD NOT send an explicit mail to normal MUAs.
     --==break==--
 
 The image format SHOULD be image/jpeg or image/png.
-Note that `Chat-User-Avatar` may appear together with all other headers,
-eg. there may be a `Chat-User-Avatar` and a `Chat-Group-Avatar` header
+Note that `Chat-Profile-Image` may appear together with all other headers,
+eg. there may be a `Chat-Profile-Image` and a `Chat-Group-Image` header
 in the same message.
-To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
+To save data, it is RECOMMENDED to add a `Chat-Profile-Image` header
 only on image changes.
 
 
-# Locations
-
-Locations can be attachted to messages using
-[standard kml-files](https://www.opengeospatial.org/standards/kml/)
-with well-known names.
-
-
-## User locations
-
-To send the location of the sender,
-the app can attach a file with the name `location.kml`.
-The file can contain one or more locations.
-Apps that support location streaming will typically collect some location events
-and send them together in one file.
-As each location has an independent timestamp,
-the apps can show the location as a track.
-
-Note that the `addr` attribute inside the  `location.kml` file
-MUST match the users email-address.
-Otherwise, the file is discarded silently;
-this is to protect against getting wrong locations,
-eg. forwarded from a normal MUA.
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <kml xmlns="http://www.opengis.net/kml/2.2">
-      <Document addr="ndh@deltachat.de">
-        <Placemark>
-          <Timestamp><when>2020-01-11T20:40:19Z</when></Timestamp>
-          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
-        </Placemark>
-        <Placemark>
-          <Timestamp><when>2020-01-11T20:40:25Z</when></Timestamp>
-          <Point><coordinates accuracy="5.4">7.654,3.21</coordinates></Point>
-        </Placemark>
-      </Document>
-    </kml>
-
-
-## Points of interest
-
-To send an "Point of interest", a POI,
-use a normal message and attach a file with the name  `message.kml`.
-In contrast to user locations, this file should contain only one location
-and an address-attribute is not needed -
-as the location belongs to the message content,
-it is fine if the location is detected on forwarding etc.
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <kml xmlns="http://www.opengis.net/kml/2.2">
-      <Document>
-        <Placemark>
-          <Timestamp><when>2020-01-01T20:40:19Z</when></Timestamp>
-          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
-        </Placemark>
-      </Document>
-    </kml>
-
-
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -438,4 +368,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2020 Delta Chat contributors.
+Copyright © 2017-2019 Delta Chat contributors.

src/aheader.rs

@@ -6,10 +6,10 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
+use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
-use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::key::{DcKey, SignedPublicKey};
+use crate::key::*;
 
 /// Possible values for encryption preference
 #[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive)]
@@ -52,17 +52,13 @@ impl str::FromStr for EncryptPreference {
 #[derive(Debug)]
 pub struct Aheader {
     pub addr: String,
-    pub public_key: SignedPublicKey,
+    pub public_key: Key,
     pub prefer_encrypt: EncryptPreference,
 }
 
 impl Aheader {
     /// Creates new autocrypt header
-    pub fn new(
-        addr: String,
-        public_key: SignedPublicKey,
-        prefer_encrypt: EncryptPreference,
-    ) -> Self {
+    pub fn new(addr: String, public_key: Key, prefer_encrypt: EncryptPreference) -> Self {
         Aheader {
             addr,
             public_key,
@@ -75,7 +71,9 @@ impl Aheader {
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
     ) -> Option<Self> {
-        if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
+        use mailparse::MailHeaderMap;
+
+        if let Ok(Some(value)) = headers.get_first_value("Autocrypt") {
             match Self::from_str(&value) {
                 Ok(header) => {
                     if addr_cmp(&header.addr, wanted_from) {
@@ -127,10 +125,14 @@ impl str::FromStr for Aheader {
             .split(';')
             .filter_map(|a| {
                 let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
-                match &attribute[..] {
-                    [key, value] => Some((key.trim().to_string(), value.trim().to_string())),
-                    _ => None,
+                if attribute.len() < 2 {
+                    return None;
                 }
+
+                Some((
+                    attribute[0].trim().to_string(),
+                    attribute[1].trim().to_string(),
+                ))
             })
             .collect();
 
@@ -140,11 +142,22 @@ impl str::FromStr for Aheader {
                 return Err(());
             }
         };
-        let public_key: SignedPublicKey = attributes
+
+        let public_key = match attributes
             .remove("keydata")
-            .ok_or(())
-            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
-            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
+            .and_then(|raw| Key::from_base64(&raw, KeyType::Public))
+        {
+            Some(key) => {
+                if key.verify() {
+                    key
+                } else {
+                    return Err(());
+                }
+            }
+            None => {
+                return Err(());
+            }
+        };
 
         let prefer_encrypt = attributes
             .remove("prefer-encrypt")
@@ -279,7 +292,7 @@ mod tests {
             "{}",
             Aheader::new(
                 "test@example.com".to_string(),
-                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                Key::from_base64(RAWKEY, KeyType::Public).unwrap(),
                 EncryptPreference::Mutual
             )
         )
@@ -292,7 +305,7 @@ mod tests {
             "{}",
             Aheader::new(
                 "test@example.com".to_string(),
-                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                Key::from_base64(RAWKEY, KeyType::Public).unwrap(),
                 EncryptPreference::NoPreference
             )
         )

src/blob.rs

@@ -6,10 +6,6 @@ use std::fs;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
-use image::GenericImageView;
-use thiserror::Error;
-
-use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
 use crate::events::Event;
 
@@ -56,6 +52,7 @@ impl<'a> BlobObject<'a> {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
                 cause: err,
+                backtrace: failure::Backtrace::new(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -83,6 +80,7 @@ impl<'a> BlobObject<'a> {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
+                            backtrace: failure::Backtrace::new(),
                         });
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -95,6 +93,7 @@ impl<'a> BlobObject<'a> {
             blobdir: dir.to_path_buf(),
             blobname: name,
             cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+            backtrace: failure::Backtrace::new(),
         })
     }
 
@@ -119,6 +118,7 @@ impl<'a> BlobObject<'a> {
             blobname: String::from(""),
             src: src.as_ref().to_path_buf(),
             cause: err,
+            backtrace: failure::Backtrace::new(),
         })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
@@ -134,6 +134,7 @@ impl<'a> BlobObject<'a> {
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
+                backtrace: failure::Backtrace::new(),
             }
         })?;
         let blob = BlobObject {
@@ -158,7 +159,7 @@ impl<'a> BlobObject<'a> {
     /// This merely delegates to the [BlobObject::create_and_copy] and
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
-    pub fn new_from_path(
+    pub fn create_from_path(
         context: &Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject, BlobError> {
@@ -193,14 +194,17 @@ impl<'a> BlobObject<'a> {
             .map_err(|_| BlobError::WrongBlobdir {
                 blobdir: context.get_blobdir().to_path_buf(),
                 src: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
             })?;
         if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
                 blobname: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
             blobname: path.as_ref().to_path_buf(),
+            backtrace: failure::Backtrace::new(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -228,6 +232,7 @@ impl<'a> BlobObject<'a> {
         if !BlobObject::is_acceptible_blob_name(&name) {
             return Err(BlobError::WrongName {
                 blobname: PathBuf::from(name),
+                backtrace: failure::Backtrace::new(),
             });
         }
         Ok(BlobObject {
@@ -313,12 +318,13 @@ impl<'a> BlobObject<'a> {
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
         let mut iter = clean.splitn(2, '.');
-        let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
-        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
-        if ext.is_empty() {
-            (stem, "".to_string())
-        } else {
-            (stem, format!(".{}", ext).to_lowercase())
+        let mut stem = iter.next().unwrap_or_default().to_string();
+        let mut ext = iter.next().unwrap_or_default().to_string();
+        stem.truncate(64);
+        ext.truncate(32);
+        match ext.len() {
+            0 => (stem, "".to_string()),
+            _ => (stem, format!(".{}", ext).to_lowercase()),
         }
     }
 
@@ -343,29 +349,6 @@ impl<'a> BlobObject<'a> {
         }
         true
     }
-
-    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
-        let blob_abs = self.to_abs_path();
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
-
-        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
-            return Ok(());
-        }
-
-        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
-
-        Ok(())
-    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -375,41 +358,88 @@ impl<'a> fmt::Display for BlobObject<'a> {
 }
 
 /// Errors for the [BlobObject].
-#[derive(Debug, Error)]
+#[derive(Fail, Debug)]
 pub enum BlobError {
-    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
     CreateFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
     WriteFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
         blobdir: PathBuf,
         blobname: String,
         src: PathBuf,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
-    RecodeFailure {
+    WrongBlobdir {
         blobdir: PathBuf,
-        blobname: String,
-        #[source]
-        cause: image::ImageError,
+        src: PathBuf,
+        backtrace: failure::Backtrace,
     },
-    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
-    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
-    #[error("Blob has a badname {}", .blobname.display())]
-    WrongName { blobname: PathBuf },
+    WrongName {
+        blobname: PathBuf,
+        backtrace: failure::Backtrace,
+    },
+}
+
+// Implementing Display is done by hand because the failure
+// #[fail(display = "...")] syntax does not allow using
+// `blobdir.display()`.
+impl fmt::Display for BlobError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // Match on the data rather than kind, they are equivalent for
+        // identifying purposes but contain the actual data we need.
+        match &self {
+            BlobError::CreateFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to create blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::WriteFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to write data to blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::CopyFailure {
+                blobdir,
+                blobname,
+                src,
+                ..
+            } => write!(
+                f,
+                "Failed to copy data from {} to blob {} in {}",
+                src.display(),
+                blobname,
+                blobdir.display(),
+            ),
+            BlobError::WrongBlobdir { blobdir, src, .. } => write!(
+                f,
+                "File path {} is not in blobdir {}",
+                src.display(),
+                blobdir.display(),
+            ),
+            BlobError::WrongName { blobname, .. } => {
+                write!(f, "Blob has a bad name: {}", blobname.display(),)
+            }
+        }
+    }
 }
 
 #[cfg(test)]
@@ -529,14 +559,14 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
 
         let src_int = t.ctx.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_int).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
@@ -546,7 +576,7 @@ mod tests {
         let t = dummy_context();
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -562,11 +592,4 @@ mod tests {
         assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
         assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
     }
-
-    #[test]
-    fn test_sanitise_name() {
-        let (_, ext) =
-            BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
-        assert_eq!(ext, ".txt");
-    }
 }

src/chatlist.rs

@@ -1,11 +1,10 @@
 //! # Chat list module
 
-use crate::chat;
 use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
-use crate::error::{bail, ensure, Result};
+use crate::error::Result;
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
@@ -37,7 +36,7 @@ use crate::stock::StockMessage;
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(ChatId, MsgId)>,
+    ids: Vec<(u32, MsgId)>,
 }
 
 impl Chatlist {
@@ -61,7 +60,7 @@ impl Chatlist {
     ///   or "Not now".
     ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
-    ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
+    ///   archived *any* chat using dc_archive_chat(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
     ///   list of all archived chats that can be created by this function hen using
     ///   the DC_GCL_ARCHIVED_ONLY flag.
@@ -74,9 +73,6 @@ impl Chatlist {
     ///   if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
     ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
-    /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
-    ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -92,21 +88,10 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
-        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
-        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
-        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
-
-        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context) {
-            warn!(context, "Failed to hide expired messages: {}", err);
-        }
-
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
-            let chat_id: ChatId = row.get(0)?;
+            let chat_id: u32 = row.get(0)?;
             let msg_id: MsgId = row.get(1).unwrap_or_default();
             Ok((chat_id, msg_id))
         };
@@ -116,14 +101,6 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        let skip_id = if flag_for_forwarding {
-            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-                .unwrap_or_default()
-                .0
-        } else {
-            ChatId::new(0)
-        };
-
         // select with left join and minimum:
         //
         // - the inner select must use `hidden` and _not_ `m.hidden`
@@ -150,21 +127,18 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?))
                  WHERE c.id>9
                    AND c.blocked=0
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, query_contact_id as i32],
                 process_row,
                 process_rows,
             )?
-        } else if flag_archived_only {
+        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
-            // (this includes the archived device-chat; we could skip it,
-            // however, then the number of archived chats do not match, which might be even more irritating.
-            // and adapting the number requires larger refactorings and seems not to be worth the effort)
             context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
@@ -188,12 +162,6 @@ impl Chatlist {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
 
-            // allow searching over special names that may change at any time
-            // when the ui calls set_stock_translation()
-            if let Err(err) = update_special_chat_names(context) {
-                warn!(context, "cannot update special chat names: {:?}", err)
-            }
-
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
                 "SELECT c.id, m.id
@@ -204,25 +172,18 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
-                 WHERE c.id>9 AND c.id!=?2
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
                    AND c.blocked=0
-                   AND c.name LIKE ?3
+                   AND c.name LIKE ?
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, str_like_cmd],
+                params![MessageState::OutDraft, str_like_cmd],
                 process_row,
                 process_rows,
             )?
         } else {
             //  show normal chatlist
-            let sort_id_up = if flag_for_forwarding {
-                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-                    .unwrap_or_default()
-                    .0
-            } else {
-                ChatId::new(0)
-            };
             let mut ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
@@ -232,24 +193,19 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
-                 WHERE c.id>9 AND c.id!=?2
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
                    AND c.blocked=0
-                   AND NOT c.archived=?3
+                   AND c.archived=0
                  GROUP BY c.id
-                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft],
                 process_row,
                 process_rows,
             )?;
-            if !flag_no_specials {
+            if 0 == listflags & DC_GCL_NO_SPECIALS {
                 if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    if !flag_for_forwarding {
-                        ids.insert(
-                            0,
-                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
-                        );
-                    }
+                    ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
                 }
                 add_archived_link_item = true;
             }
@@ -257,10 +213,10 @@ impl Chatlist {
         };
 
         if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
-            if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
+            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
             }
-            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -279,21 +235,19 @@ impl Chatlist {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> ChatId {
-        match self.ids.get(index) {
-            Some((chat_id, _msg_id)) => *chat_id,
-            None => ChatId::new(0),
+    pub fn get_chat_id(&self, index: usize) -> u32 {
+        if index >= self.ids.len() {
+            return 0;
         }
+        self.ids[index].0
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
-        match self.ids.get(index) {
-            Some((_chat_id, msg_id)) => Ok(*msg_id),
-            None => bail!("Chatlist index out of range"),
-        }
+        ensure!(index < self.ids.len(), "Chatlist index out of range");
+        Ok(self.ids[index].1)
     }
 
     /// Get a summary for a chatlist index.
@@ -315,29 +269,27 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let mut ret = Lot::new();
 
-        let (chat_id, lastmsg_id) = match self.ids.get(index) {
-            Some(ids) => ids,
-            None => {
-                ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return ret;
-            }
-        };
+        let mut ret = Lot::new();
+        if index >= self.ids.len() {
+            ret.text2 = Some("ErrBadChatlistIndex".to_string());
+            return ret;
+        }
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) {
+        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
             chat_loaded = chat;
             &chat_loaded
         } else {
             return ret;
         };
 
+        let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
@@ -349,7 +301,7 @@ impl Chatlist {
             None
         };
 
-        if chat.id.is_archived_link() {
+        if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
@@ -360,13 +312,9 @@ impl Chatlist {
 
         ret
     }
-
-    pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
-        self.ids.iter().position(|(chat_id, _)| chat_id == &id)
-    }
 }
 
-/// Returns the number of archived chats
+/// Get the number of archived chats
 pub fn dc_get_archived_cnt(context: &Context) -> u32 {
     context
         .sql
@@ -401,6 +349,7 @@ fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
 mod tests {
     use super::*;
 
+    use crate::chat;
     use crate::test_utils::*;
 
     #[test]
@@ -420,7 +369,7 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg));
+        set_draft(&t.ctx, chat_id2, Some(&mut msg));
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
@@ -431,66 +380,8 @@ mod tests {
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 0);
 
-        chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
-            .ok();
+        chat::archive(&t.ctx, chat_id1, true).ok();
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 1);
     }
-
-    #[test]
-    fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context();
-        t.ctx.update_device_chats().unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
-            .unwrap()
-            .is_self_talk());
-
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None).unwrap();
-        assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
-            .unwrap()
-            .is_self_talk());
-    }
-
-    #[test]
-    fn test_search_special_chat_names() {
-        let t = dummy_context();
-        t.ctx.update_device_chats().unwrap();
-
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
-        assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
-        assert_eq!(chats.len(), 0);
-
-        t.ctx
-            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
-            .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
-        assert_eq!(chats.len(), 1);
-
-        t.ctx
-            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
-            .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
-        assert_eq!(chats.len(), 1);
-    }
-
-    #[test]
-    fn test_get_summary_unwrap() {
-        let t = dummy_context();
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg));
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None);
-        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
-    }
 }

src/config.rs

@@ -4,16 +4,11 @@ use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
-use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::events::Event;
 use crate::job::*;
-use crate::message::MsgId;
-use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
-use rusqlite::NO_PARAMS;
 
 /// The available configuration keys.
 #[derive(
@@ -65,31 +60,6 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
-    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
-    MediaQuality,
-
-    #[strum(props(default = "0"))]
-    KeyGenType,
-
-    /// Timer in seconds after which the message is deleted from the
-    /// server.
-    ///
-    /// Equals to 0 by default, which means the message is never
-    /// deleted.
-    ///
-    /// Value 1 is treated as "delete at once": messages are deleted
-    /// immediately, without moving to DeltaChat folder.
-    #[strum(props(default = "0"))]
-    DeleteServerAfter,
-
-    /// Timer in seconds after which the message is deleted from the
-    /// device.
-    ///
-    /// Equals to 0 by default, which means the message is never
-    /// deleted.
-    #[strum(props(default = "0"))]
-    DeleteDeviceAfter,
-
     SaveMimeHeaders,
     ConfiguredAddr,
     ConfiguredMailServer,
@@ -127,7 +97,7 @@ impl Context {
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
-            Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
+            Config::SysMsgsizeMaxRecommended => Some(format!("{}", 24 * 1024 * 1024 / 4 * 3)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
             _ => self.sql.get_raw_config(self, key),
         };
@@ -153,50 +123,17 @@ impl Context {
         self.get_config_int(key) != 0
     }
 
-    /// Gets configured "delete_server_after" value.
-    ///
-    /// `None` means never delete the message, `Some(0)` means delete
-    /// at once, `Some(x)` means delete after `x` seconds.
-    pub fn get_config_delete_server_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteServerAfter) {
-            0 => None,
-            1 => Some(0),
-            x => Some(x as i64),
-        }
-    }
-
-    /// Gets configured "delete_device_after" value.
-    ///
-    /// `None` means never delete the message, `Some(x)` means delete
-    /// after `x` seconds.
-    pub fn get_config_delete_device_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteDeviceAfter) {
-            0 => None,
-            x => Some(x as i64),
-        }
-    }
-
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
     pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
-            Config::Selfavatar => {
-                self.sql
-                    .execute("UPDATE contacts SET selfavatar_sent=0;", NO_PARAMS)?;
-                self.sql
-                    .set_raw_config_bool(self, "attach_selfavatar", true)?;
-                match value {
-                    Some(value) => {
-                        let blob = BlobObject::new_from_path(&self, value)?;
-                        blob.recode_to_avatar_size(self)?;
-                        self.sql.set_raw_config(self, key, Some(blob.as_name()))
-                    }
-                    None => self.sql.set_raw_config(self, key, None),
-                }
+            Config::Selfavatar if value.is_some() => {
+                let blob = BlobObject::create_from_path(&self, value.unwrap())?;
+                self.sql.set_raw_config(self, key, Some(blob.as_name()))
             }
             Config::InboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_inbox_idle(self);
+                interrupt_inbox_idle(self, true);
                 ret
             }
             Config::SentboxWatch => {
@@ -219,15 +156,6 @@ impl Context {
 
                 self.sql.set_raw_config(self, key, val)
             }
-            Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                // Force chatlist reload to delete old messages immediately.
-                self.call_cb(Event::MsgsChanged {
-                    msg_id: MsgId::new(0),
-                    chat_id: ChatId::new(0),
-                });
-                ret
-            }
             _ => self.sql.set_raw_config(self, key, value),
         }
     }
@@ -251,13 +179,7 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
-    use crate::constants;
-    use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
-    use image::GenericImageView;
-    use num_traits::FromPrimitive;
-    use std::fs::File;
-    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -277,95 +199,30 @@ mod tests {
     }
 
     #[test]
-    fn test_selfavatar_outside_blobdir() {
+    fn test_selfavatar() -> failure::Fallible<()> {
         let t = dummy_context();
         let avatar_src = t.dir.path().join("avatar.jpg");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
+        std::fs::write(&avatar_src, b"avatar")?;
         let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
         assert!(!avatar_blob.exists());
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .unwrap();
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
         assert!(avatar_blob.exists());
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        assert_eq!(std::fs::read(&avatar_blob)?, b"avatar");
         let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        Ok(())
     }
 
     #[test]
-    fn test_selfavatar_in_blobdir() {
+    fn test_selfavatar_in_blobdir() -> failure::Fallible<()> {
         let t = dummy_context();
-        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
+        let avatar_src = t.ctx.get_blobdir().join("avatar.jpg");
+        std::fs::write(&avatar_src, b"avatar")?;
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .unwrap();
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
         let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
-    }
-
-    #[test]
-    fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context();
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists());
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .unwrap();
-        assert!(avatar_blob.exists());
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
-    #[test]
-    fn test_media_quality_config_option() {
-        let t = dummy_context();
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
-        assert_eq!(media_quality, 0);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Balanced);
-
-        t.ctx.set_config(Config::MediaQuality, Some("1")).unwrap();
-
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
-        assert_eq!(media_quality, 1);
-        assert_eq!(constants::MediaQuality::Worse as i32, 1);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Worse);
+        Ok(())
     }
 }

src/configure/auto_mozilla.rs

@@ -1,6 +1,7 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
 //! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
+use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
@@ -8,7 +9,33 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-use super::Error;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "Invalid email address: {:?}", _0)]
+    InvalidEmailAddress(String),
+
+    #[fail(display = "XML error at position {}", position)]
+    InvalidXml {
+        position: usize,
+        #[cause]
+        error: quick_xml::Error,
+    },
+
+    #[fail(display = "Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[fail(display = "Failed to get URL {}", _0)]
+    ReadUrlError(#[cause] super::read_url::Error),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<super::read_url::Error> for Error {
+    fn from(err: super::read_url::Error) -> Error {
+        Error::ReadUrlError(err)
+    }
+}
 
 #[derive(Debug)]
 struct MozAutoconfigure<'a> {
@@ -38,16 +65,16 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
+fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     // Split address into local part and domain part.
-    let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
-    let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
-        [domain, local] => (local, domain),
-        _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
-    };
+    let p = in_emailaddr
+        .find('@')
+        .ok_or_else(|| Error::InvalidEmailAddress(in_emailaddr.to_string()))?;
+    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
+    let in_emaildomain = &in_emaildomain[1..];
 
     let mut moz_ac = MozAutoconfigure {
         in_emailaddr,
@@ -98,7 +125,7 @@ pub fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<LoginParam> {
     let xml_raw = read_url(context, url)?;
 
     let res = parse_xml(&param_in.addr, &xml_raw);

src/configure/auto_outlook.rs

@@ -1,5 +1,6 @@
 //! Outlook's Autodiscover
 
+use quick_xml;
 use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
@@ -7,7 +8,33 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-use super::Error;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "XML error at position {}", position)]
+    InvalidXml {
+        position: usize,
+        #[cause]
+        error: quick_xml::Error,
+    },
+
+    #[fail(display = "Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[fail(display = "Failed to get URL {}", _0)]
+    ReadUrlError(#[cause] super::read_url::Error),
+
+    #[fail(display = "Number of redirection is exceeded")]
+    RedirectionError,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<super::read_url::Error> for Error {
+    fn from(err: super::read_url::Error) -> Error {
+        Error::ReadUrlError(err)
+    }
+}
 
 struct OutlookAutodiscover {
     pub out: LoginParam,
@@ -25,7 +52,7 @@ enum ParsingResult {
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult> {
     let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
         out_imap_set: false,
@@ -116,7 +143,7 @@ pub fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<LoginParam> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {

src/configure/mod.rs

@@ -6,20 +6,16 @@ mod read_url;
 
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use async_std::task;
-
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::format_err;
-use crate::job::{self, job_add, job_kill_action};
-use crate::login_param::{CertificateChecks, LoginParam};
+use crate::e2ee;
+use crate::job::*;
+use crate::login_param::LoginParam;
 use crate::oauth2::*;
 use crate::param::Params;
-use crate::{chat, e2ee, provider};
 
-use crate::message::Message;
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
 
@@ -33,43 +29,40 @@ macro_rules! progress {
     };
 }
 
-impl Context {
-    /// Starts a configuration job.
-    pub fn configure(&self) {
-        if self.has_ongoing() {
-            warn!(self, "There is already another ongoing process running.",);
-            return;
-        }
-        job_kill_action(self, job::Action::ConfigureImap);
-        job_add(self, job::Action::ConfigureImap, 0, Params::new(), 0);
+// connect
+pub fn configure(context: &Context) {
+    if context.has_ongoing() {
+        warn!(context, "There is already another ongoing process running.",);
+        return;
     }
+    job_kill_action(context, Action::ConfigureImap);
+    job_add(context, Action::ConfigureImap, 0, Params::new(), 0);
+}
 
-    /// Checks if the context is already configured.
-    pub fn is_configured(&self) -> bool {
-        self.sql.get_raw_config_bool(self, "configured")
-    }
+/// Check if the context is already configured.
+pub fn dc_is_configured(context: &Context) -> bool {
+    context.sql.get_raw_config_bool(context, "configured")
 }
 
 /*******************************************************************************
  * Configure JOB
  ******************************************************************************/
-#[allow(non_snake_case, unused_must_use, clippy::cognitive_complexity)]
-pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
+#[allow(non_snake_case, unused_must_use)]
+pub fn JobConfigureImap(context: &Context) {
     if !context.sql.is_open() {
         error!(context, "Cannot configure, database not opened.",);
         progress!(context, 0);
-        return job::Status::Finished(Err(format_err!("Database not opened")));
+        return;
     }
     if !context.alloc_ongoing() {
         progress!(context, 0);
-        return job::Status::Finished(Err(format_err!("Cannot allocated ongoing process")));
+        return;
     }
     let mut success = false;
     let mut imap_connected_here = false;
     let mut smtp_connected_here = false;
 
     let mut param_autoconfig: Option<LoginParam> = None;
-    let was_configured_before = context.is_configured();
 
     context
         .inbox_thread
@@ -99,11 +92,9 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     let mut param_domain = "undefined.undefined".to_owned();
     let mut param_addr_urlencoded: String =
         "Internal Error: this value should never be used".to_owned();
-    let mut keep_flags = 0;
-
-    const STEP_12_USE_AUTOCONFIG: u8 = 12;
-    const STEP_13_AFTER_AUTOCONFIG: u8 = 13;
+    let mut keep_flags = std::i32::MAX;
 
+    const STEP_3_INDEX: u8 = 13;
     let mut step_counter: u8 = 0;
     while !context.shall_stop_ongoing() {
         step_counter += 1;
@@ -119,7 +110,7 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
             }
             // Step 1: Load the parameters and check email-address and password
             2 => {
-                if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
+                if 0 != param.server_flags & 0x2 {
                     // the used oauth2 addr may differ, check this.
                     // if dc_get_oauth2_addr() is not available in the oauth2 implementation,
                     // just use the given one.
@@ -154,7 +145,6 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
             // Step 2: Autoconfig
             4 => {
                 progress!(context, 200);
-
                 if param.mail_server.is_empty()
                             && param.mail_port == 0
                             /*&&param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then */
@@ -162,18 +152,12 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
                             && param.send_port == 0
                             && param.send_user.is_empty()
                             /*&&param.send_pw.is_empty() -- the password cannot be auto-configured and is no criterion for autoconfig or not */
-                            && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
+                            && param.server_flags & !0x2 == 0
                 {
-                    // no advanced parameters entered by the user: query provider-database or do Autoconfig
-                    keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
-                    if let Some(new_param) = get_offline_autoconfig(context, &param) {
-                        // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
-                        param_autoconfig = Some(new_param);
-                        step_counter = STEP_12_USE_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
-                    }
+                    keep_flags = param.server_flags & 0x2;
                 } else {
-                    // advanced parameters entered by the user: skip Autoconfig
-                    step_counter = STEP_13_AFTER_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
+                    // Autoconfig is not needed so skip it.
+                    step_counter = STEP_3_INDEX - 1;
                 }
                 true
             }
@@ -205,7 +189,10 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
             7 => {
                 progress!(context, 310);
                 if param_autoconfig.is_none() {
-                    let url = format!("https://{}/autodiscover/autodiscover.xml", param_domain);
+                    let url = format!(
+                        "https://{}{}/autodiscover/autodiscover.xml",
+                        "", param_domain
+                    );
                     param_autoconfig = outlk_autodiscover(context, &url, &param).ok();
                 }
                 true
@@ -255,10 +242,8 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
                 }
                 true
             }
-            /* C.  Do we have any autoconfig result?
-               If you change the match-number here, also update STEP_12_COPY_AUTOCONFIG above
-            */
-            STEP_12_USE_AUTOCONFIG => {
+            /* C.  Do we have any result? */
+            12 => {
                 progress!(context, 500);
                 if let Some(ref cfg) = param_autoconfig {
                     info!(context, "Got autoconfig: {}", &cfg);
@@ -271,15 +256,15 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
                     param.send_port = cfg.send_port;
                     param.send_user = cfg.send_user.clone();
                     param.server_flags = cfg.server_flags;
-                    /* although param_autoconfig's data are no longer needed from,
-                    it is used to later to prevent trying variations of port/server/logins */
+                    /* although param_autoconfig's data are no longer needed from, it is important to keep the object as
+                    we may enter "deep guessing" if we could not read a configuration */
                 }
                 param.server_flags |= keep_flags;
                 true
             }
             // Step 3: Fill missing fields with defaults
-            // If you change the match-number here, also update STEP_13_AFTER_AUTOCONFIG above
-            STEP_13_AFTER_AUTOCONFIG => {
+            13 => {
+                // if you move this, don't forget to update STEP_3_INDEX, too
                 if param.mail_server.is_empty() {
                     param.mail_server = format!("imap.{}", param_domain,)
                 }
@@ -373,7 +358,7 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
                 let create_mvbox = context.get_config_bool(Config::MvboxWatch)
                     || context.get_config_bool(Config::MvboxMove);
                 let imap = &context.inbox_thread.read().unwrap().imap;
-                if let Err(err) = imap.configure_folders(context, create_mvbox) {
+                if let Err(err) = imap.ensure_configured_folders(context, create_mvbox) {
                     warn!(context, "configuring folders failed: {:?}", err);
                     false
                 } else {
@@ -441,86 +426,8 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
         LoginParam::from_database(context, "configured_raw_").save_to_database(context, "");
     }
 
-    if let Some(provider) = provider::get_provider_info(&param.addr) {
-        if !was_configured_before {
-            if let Some(config_defaults) = &provider.config_defaults {
-                for def in config_defaults.iter() {
-                    info!(context, "apply config_defaults {}={}", def.key, def.value);
-                    context.set_config(def.key, Some(def.value));
-                }
-            }
-        }
-
-        if !provider.after_login_hint.is_empty() {
-            let mut msg = Message::new(Viewtype::Text);
-            msg.text = Some(provider.after_login_hint.to_string());
-            if chat::add_device_msg(context, Some("core-provider-info"), Some(&mut msg)).is_err() {
-                warn!(context, "cannot add after_login_hint as core-provider-info");
-            }
-        }
-    }
-
     context.free_ongoing();
     progress!(context, if success { 1000 } else { 0 });
-    job::Status::Finished(Ok(()))
-}
-
-#[allow(clippy::unnecessary_unwrap)]
-fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
-    info!(
-        context,
-        "checking internal provider-info for offline autoconfig"
-    );
-
-    if let Some(provider) = provider::get_provider_info(&param.addr) {
-        match provider.status {
-            provider::Status::OK | provider::Status::PREPARATION => {
-                let imap = provider.get_imap_server();
-                let smtp = provider.get_smtp_server();
-                // clippy complains about these is_some()/unwrap() settings,
-                // however, rewriting the code to "if let" would make things less obvious,
-                // esp. if we allow more combinations of servers (pop, jmap).
-                // therefore, #[allow(clippy::unnecessary_unwrap)] is added above.
-                if imap.is_some() && smtp.is_some() {
-                    let imap = imap.unwrap();
-                    let smtp = smtp.unwrap();
-
-                    let mut p = LoginParam::new();
-                    p.addr = param.addr.clone();
-
-                    p.mail_server = imap.hostname.to_string();
-                    p.mail_user = imap.apply_username_pattern(param.addr.clone());
-                    p.mail_port = imap.port as i32;
-                    p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                    p.server_flags |= match imap.socket {
-                        provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
-                        provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
-                    };
-
-                    p.send_server = smtp.hostname.to_string();
-                    p.send_user = smtp.apply_username_pattern(param.addr.clone());
-                    p.send_port = smtp.port as i32;
-                    p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                    p.server_flags |= match smtp.socket {
-                        provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
-                        provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
-                    };
-
-                    info!(context, "offline autoconfig found: {}", p);
-                    return Some(p);
-                } else {
-                    info!(context, "offline autoconfig found, but no servers defined");
-                    return None;
-                }
-            }
-            provider::Status::BROKEN => {
-                info!(context, "offline autoconfig found, provider is broken");
-                return None;
-            }
-        }
-    }
-    info!(context, "no offline autoconfig found");
-    None
 }
 
 fn try_imap_connections(
@@ -578,22 +485,17 @@ fn try_imap_connection(
 
 fn try_imap_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
     let inf = format!(
-        "imap: {}@{}:{} flags=0x{:x} certificate_checks={}",
-        param.mail_user,
-        param.mail_server,
-        param.mail_port,
-        param.server_flags,
-        param.imap_certificate_checks
+        "imap: {}@{}:{} flags=0x{:x}",
+        param.mail_user, param.mail_server, param.mail_port, param.server_flags
     );
     info!(context, "Trying: {}", inf);
-    if task::block_on(
-        context
-            .inbox_thread
-            .read()
-            .unwrap()
-            .imap
-            .connect(context, &param),
-    ) {
+    if context
+        .inbox_thread
+        .read()
+        .unwrap()
+        .imap
+        .connect(context, &param)
+    {
         info!(context, "success: {}", inf);
         return Some(true);
     }
@@ -662,32 +564,9 @@ fn try_smtp_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
     }
 }
 
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("Invalid email address: {0:?}")]
-    InvalidEmailAddress(String),
-
-    #[error("XML error at position {position}")]
-    InvalidXml {
-        position: usize,
-        #[source]
-        error: quick_xml::Error,
-    },
-
-    #[error("Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[error("Failed to get URL")]
-    ReadUrlError(#[from] self::read_url::Error),
-
-    #[error("Number of redirection is exceeded")]
-    RedirectionError,
-}
-
 #[cfg(test)]
 mod tests {
 
-    use super::*;
     use crate::config::*;
     use crate::configure::JobConfigureImap;
     use crate::test_utils::*;
@@ -701,19 +580,4 @@ mod tests {
         t.ctx.set_config(Config::MailPw, Some("123456")).unwrap();
         JobConfigureImap(&t.ctx);
     }
-
-    #[test]
-    fn test_get_offline_autoconfig() {
-        let context = dummy_context().ctx;
-
-        let mut params = LoginParam::new();
-        params.addr = "someone123@example.org".to_string();
-        assert!(get_offline_autoconfig(&context, &params).is_none());
-
-        let mut params = LoginParam::new();
-        params.addr = "someone123@nauta.cu".to_string();
-        let found_params = get_offline_autoconfig(&context, &params).unwrap();
-        assert_eq!(found_params.mail_server, "imap.nauta.cu".to_string());
-        assert_eq!(found_params.send_server, "smtp.nauta.cu".to_string());
-    }
 }

src/configure/read_url.rs

@@ -1,18 +1,20 @@
 use crate::context::Context;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("URL request error")]
-    GetError(#[from] reqwest::Error),
+    #[fail(display = "URL request error")]
+    GetError(#[cause] reqwest::Error),
 }
 
-pub fn read_url(context: &Context, url: &str) -> Result<String, Error> {
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub fn read_url(context: &Context, url: &str) -> Result<String> {
     info!(context, "Requesting URL {}", url);
 
-    match reqwest::blocking::Client::new()
+    match reqwest::Client::new()
         .get(url)
         .send()
-        .and_then(|res| res.text())
+        .and_then(|mut res| res.text())
     {
         Ok(res) => Ok(res),
         Err(err) => {

src/constants.rs

@@ -1,9 +1,8 @@
 //! # Constants
-#![allow(dead_code)]
+#![allow(non_camel_case_types, dead_code)]
 
 use deltachat_derive::*;
 use lazy_static::lazy_static;
-use serde::{Deserialize, Serialize};
 
 lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
@@ -16,20 +15,7 @@ const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
 
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum Blocked {
     Not = 0,
@@ -57,51 +43,24 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
-#[repr(u8)]
-pub enum MediaQuality {
-    Balanced = 0,
-    Worse = 1,
-}
-
-impl Default for MediaQuality {
-    fn default() -> Self {
-        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
-    }
-}
-
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
-#[repr(u8)]
-pub enum KeyGenType {
-    Default = 0,
-    Rsa2048 = 1,
-    Ed25519 = 2,
-}
-
-impl Default for KeyGenType {
-    fn default() -> Self {
-        KeyGenType::Default
-    }
-}
+pub const DC_IMAP_SEEN: u32 = 0x1;
 
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
 
-pub(crate) const DC_FROM_HANDSHAKE: i32 = 0x01;
-
 pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
-pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
 
 pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
 pub const DC_GCL_ADD_SELF: usize = 0x02;
 
-// unchanged user avatars are resent to the recipients every some days
-pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
+// values for DC_PARAM_FORCE_PLAINTEXT
+pub(crate) const DC_FP_NO_AUTOCRYPT_HEADER: i32 = 2;
+pub(crate) const DC_FP_ADD_AUTOCRYPT_HEADER: i32 = 1;
 
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
@@ -130,8 +89,6 @@ pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
     FromSql,
     ToSql,
     IntoStaticStr,
-    Serialize,
-    Deserialize,
 )]
 #[repr(u32)]
 pub enum Chattype {
@@ -154,7 +111,7 @@ pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 /// approx. max. length returned by dc_msg_get_text()
 const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100_000;
+const DC_MAX_GET_INFO_LEN: usize = 100000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -162,10 +119,6 @@ pub const DC_CONTACT_ID_INFO: u32 = 2;
 pub const DC_CONTACT_ID_DEVICE: u32 = 5;
 pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
 
-// decorative address that is used for DC_CONTACT_ID_DEVICE
-// when an api that returns an email is called.
-pub const DC_CONTACT_ID_DEVICE_ADDR: &str = "device@localhost";
-
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;
@@ -210,13 +163,13 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
 pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
 
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
+pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
+    (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
+    (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN);
 
 // QR code scanning (view from Bob, the joiner)
 pub const DC_VC_AUTH_REQUIRED: i32 = 2;
@@ -224,30 +177,10 @@ pub const DC_VC_CONTACT_CONFIRM: i32 = 6;
 pub const DC_BOB_ERROR: i32 = 0;
 pub const DC_BOB_SUCCESS: i32 = 1;
 
-// max. width/height of an avatar
-pub const AVATAR_SIZE: u32 = 192;
-
-// this value can be increased if the folder configuration is changed and must be redone on next program start
-pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
-
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i32)]
 pub enum Viewtype {
     Unknown = 0,
-
     /// Text message.
     /// The text of the message is set using dc_msg_set_text()
     /// and retrieved with dc_msg_get_text().
@@ -327,6 +260,8 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
 const DC_STR_DRAFT: usize = 3;
+const DC_STR_MEMBER: usize = 4;
+const DC_STR_CONTACT: usize = 6;
 const DC_STR_VOICEMESSAGE: usize = 7;
 const DC_STR_DEADDROP: usize = 8;
 const DC_STR_IMAGE: usize = 9;
@@ -358,6 +293,7 @@ const DC_STR_ARCHIVEDCHATS: usize = 40;
 const DC_STR_STARREDMSGS: usize = 41;
 const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
 const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
+const DC_STR_SELFTALK_SUBTITLE: usize = 50;
 const DC_STR_CANNOT_LOGIN: usize = 60;
 const DC_STR_SERVER_RESPONSE: usize = 61;
 const DC_STR_MSGACTIONBYUSER: usize = 62;

src/contact.rs

@@ -1,27 +1,27 @@
-//! Contacts module
-
 use std::path::PathBuf;
 
 use deltachat_derive::*;
 use itertools::Itertools;
+use rusqlite;
 
 use crate::aheader::EncryptPreference;
-use crate::chat::ChatId;
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, format_err, Result};
+use crate::e2ee;
+use crate::error::Result;
 use crate::events::Event;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
-use crate::mimeparser::AvatarAction;
-use crate::param::*;
 use crate::peerstate::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
+/// Contacts with at least this origin value are shown in the contact list.
+const DC_ORIGIN_MIN_CONTACT_LIST: i32 = 0x100;
+
 /// An object representing a single contact in memory.
 ///
 /// The contact object is not updated.
@@ -44,28 +44,20 @@ pub struct Contact {
     ///
     /// Normal contact IDs are larger than these special ones (larger than DC_CONTACT_ID_LAST_SPECIAL).
     pub id: u32,
-
     /// Contact name. It is recommended to use `Contact::get_name`,
     /// `Contact::get_display_name` or `Contact::get_name_n_addr` to access this field.
     /// May be empty, initially set to `authname`.
     name: String,
-
     /// Name authorized by the contact himself. Only this name may be spread to others,
     /// e.g. in To:-lists. May be empty. It is recommended to use `Contact::get_authname`,
     /// to access this field.
     authname: String,
-
-    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr` to access this field.
+    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr`` to access this field.
     addr: String,
-
     /// Blocked state. Use dc_contact_is_blocked to access this field.
-    pub blocked: bool,
-
+    blocked: bool,
     /// The origin/source of the contact.
     pub origin: Origin,
-
-    /// Parameters as Param::ProfileImage
-    pub param: Params,
 }
 
 /// Possible origins of a contact.
@@ -75,55 +67,38 @@ pub struct Contact {
 #[repr(i32)]
 pub enum Origin {
     Unknown = 0,
-
     /// From: of incoming messages of unknown sender
     IncomingUnknownFrom = 0x10,
-
     /// Cc: of incoming messages of unknown sender
     IncomingUnknownCc = 0x20,
-
     /// To: of incoming messages of unknown sender
     IncomingUnknownTo = 0x40,
-
     /// address scanned but not verified
     UnhandledQrScan = 0x80,
-
     /// Reply-To: of incoming message of known sender
-    /// Contacts with at least this origin value are shown in the contact list.
     IncomingReplyTo = 0x100,
-
     /// Cc: of incoming message of known sender
     IncomingCc = 0x200,
-
     /// additional To:'s of incoming message of known sender
     IncomingTo = 0x400,
-
     /// a chat was manually created for this user, but no message yet sent
     CreateChat = 0x800,
-
     /// message sent by us
     OutgoingBcc = 0x1000,
-
     /// message sent by us
     OutgoingCc = 0x2000,
-
     /// message sent by us
     OutgoingTo = 0x4000,
-
     /// internal use
     Internal = 0x40000,
-
     /// address is in our address book
-    AddressBook = 0x80000,
-
+    AdressBook = 0x80000,
     /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinInvited = 0x0100_0000,
-
+    SecurejoinInvited = 0x1000000,
     /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinJoined = 0x0200_0000,
-
-    /// contact added mannually by dc_create_contact(), this should be the largest origin as otherwise the user cannot modify the names
-    ManuallyCreated = 0x0400_0000,
+    SecurejoinJoined = 0x2000000,
+    /// contact added mannually by dc_create_contact(), this should be the largets origin as otherwise the user cannot modify the names
+    ManuallyCreated = 0x4000000,
 }
 
 impl Default for Origin {
@@ -133,16 +108,19 @@ impl Default for Origin {
 }
 
 impl Origin {
-    /// Contacts that are known, i. e. they came in via accepted contacts or
-    /// themselves an accepted contact. Known contacts are shown in the
-    /// contact list when one creates a chat and wants to add members etc.
-    pub fn is_known(self) -> bool {
-        self >= Origin::IncomingReplyTo
+    /// Contacts that are verified and known not to be spam.
+    pub fn is_verified(self) -> bool {
+        self as i32 >= 0x100
+    }
+
+    /// Contacts that are shown in the contact list.
+    pub fn include_in_contactlist(self) -> bool {
+        self as i32 >= DC_ORIGIN_MIN_CONTACT_LIST
     }
 }
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub(crate) enum Modifier {
+pub enum Modifier {
     None,
     Modified,
     Created,
@@ -161,10 +139,32 @@ pub enum VerifiedStatus {
 
 impl Contact {
     pub fn load_from_db(context: &Context, contact_id: u32) -> crate::sql::Result<Self> {
-        let mut res = context.sql.query_row(
-            "SELECT c.name, c.addr, c.origin, c.blocked, c.authname, c.param
-               FROM contacts c
-              WHERE c.id=?;",
+        if contact_id == DC_CONTACT_ID_SELF {
+            let contact = Contact {
+                id: contact_id,
+                name: context.stock_str(StockMessage::SelfMsg).into(),
+                authname: "".into(),
+                addr: context
+                    .get_config(Config::ConfiguredAddr)
+                    .unwrap_or_default(),
+                blocked: false,
+                origin: Origin::Unknown,
+            };
+            return Ok(contact);
+        } else if contact_id == DC_CONTACT_ID_DEVICE {
+            let contact = Contact {
+                id: contact_id,
+                name: context.stock_str(StockMessage::DeviceMessages).into(),
+                authname: "".into(),
+                addr: "device@localhost".into(),
+                blocked: false,
+                origin: Origin::Unknown,
+            };
+            return Ok(contact);
+        }
+
+        context.sql.query_row(
+            "SELECT c.name, c.addr, c.origin, c.blocked, c.authname  FROM contacts c  WHERE c.id=?;",
             params![contact_id as i32],
             |row| {
                 let contact = Self {
@@ -174,21 +174,10 @@ impl Contact {
                     addr: row.get::<_, String>(1)?,
                     blocked: row.get::<_, Option<i32>>(3)?.unwrap_or_default() != 0,
                     origin: row.get(2)?,
-                    param: row.get::<_, String>(5)?.parse().unwrap_or_default(),
                 };
                 Ok(contact)
-            },
-        )?;
-        if contact_id == DC_CONTACT_ID_SELF {
-            res.name = context.stock_str(StockMessage::SelfMsg).to_string();
-            res.addr = context
-                .get_config(Config::ConfiguredAddr)
-                .unwrap_or_default();
-        } else if contact_id == DC_CONTACT_ID_DEVICE {
-            res.name = context.stock_str(StockMessage::DeviceMessages).to_string();
-            res.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
-        }
-        Ok(res)
+            }
+        )
     }
 
     /// Returns `true` if this contact is blocked.
@@ -259,7 +248,7 @@ impl Contact {
         .is_ok()
         {
             context.call_cb(Event::MsgsChanged {
-                chat_id: ChatId::new(0),
+                chat_id: 0,
                 msg_id: MsgId::new(0),
             });
         }
@@ -270,7 +259,7 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `dc_may_be_valid_addr()`.
-    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>, min_origin: Origin) -> u32 {
+    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>) -> u32 {
         if addr.as_ref().is_empty() {
             return 0;
         }
@@ -283,43 +272,22 @@ impl Contact {
         if addr_cmp(addr_normalized, addr_self) {
             return DC_CONTACT_ID_SELF;
         }
+
         context.sql.query_get_value(
             context,
             "SELECT id FROM contacts WHERE addr=?1 COLLATE NOCASE AND id>?2 AND origin>=?3 AND blocked=0;",
             params![
                 addr_normalized,
                 DC_CONTACT_ID_LAST_SPECIAL as i32,
-                min_origin as u32,
+                DC_ORIGIN_MIN_CONTACT_LIST,
             ],
         ).unwrap_or_default()
     }
 
     /// Lookup a contact and create it if it does not exist yet.
-    /// The contact is identified by the email-address, a name and an "origin" can be given.
-    ///
-    /// The "origin" is where the address comes from -
-    /// from-header, cc-header, addressbook, qr, manual-edit etc.
-    /// In general, "better" origins overwrite the names of "worse" origins -
-    /// Eg. if we got a name in cc-header and later in from-header, the name will change -
-    /// this does not happen the other way round.
-    ///
-    /// The "best" origin are manually created contacts -
-    /// names given manually can only be overwritten by further manual edits
-    /// (until they are set empty again or reset to the name seen in the From-header).
-    ///
-    /// These manually edited names are _never_ used for sending on the wire -
-    /// this should avoid sending sth. as "Mama" or "Daddy" to some 3rd party.
-    /// Instead, for the wire, we use so called "authnames"
-    /// that can only be set and updated by a From-header.
-    ///
-    /// The different names used in the function are:
-    /// - "name": name passed as function argument, belonging to the given origin
-    /// - "row_name": current name used in the database, typically set to "name"
-    /// - "row_authname": name as authorized from a contact, set only through a From-header
-    /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
     ///
     /// Returns the contact_id and a `Modifier` value indicating if a modification occured.
-    pub(crate) fn add_or_lookup(
+    pub fn add_or_lookup(
         context: &Context,
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
@@ -373,9 +341,7 @@ impl Contact {
 
                 if !name.as_ref().is_empty() {
                     if !row_name.is_empty() {
-                        if (origin >= row_origin || row_name == row_authname)
-                            && name.as_ref() != row_name
-                        {
+                        if origin >= row_origin && name.as_ref() != row_name {
                             update_name = true;
                         }
                     } else {
@@ -384,9 +350,6 @@ impl Contact {
                     if origin == Origin::IncomingUnknownFrom && name.as_ref() != row_authname {
                         update_authname = true;
                     }
-                } else if origin == Origin::ManuallyCreated && !row_authname.is_empty() {
-                    // no name given on manual edit, this will update the name to the authname
-                    update_name = true;
                 }
 
                 Ok((row_id, row_name, row_addr, row_origin, row_authname))
@@ -397,22 +360,16 @@ impl Contact {
                 update_addr = true;
             }
             if update_name || update_authname || update_addr || origin > row_origin {
-                let new_name = if update_name {
-                    if !name.as_ref().is_empty() {
-                        name.as_ref()
-                    } else {
-                        &row_authname
-                    }
-                } else {
-                    &row_name
-                };
-
                 sql::execute(
                     context,
                     &context.sql,
                     "UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
                     params![
-                        new_name,
+                        if update_name {
+                            name.as_ref()
+                        } else {
+                            &row_name
+                        },
                         if update_addr { addr } else { &row_addr },
                         if origin > row_origin {
                             origin
@@ -430,41 +387,27 @@ impl Contact {
                 .ok();
 
                 if update_name {
-                    // Update the contact name also if it is used as a group name.
-                    // This is one of the few duplicated data, however, getting the chat list is easier this way.
                     sql::execute(
                     context,
                     &context.sql,
                     "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    params![new_name, Chattype::Single, row_id]
+                    params![name.as_ref(), Chattype::Single, row_id]
                 ).ok();
                 }
                 sth_modified = Modifier::Modified;
             }
+        } else if sql::execute(
+            context,
+            &context.sql,
+            "INSERT INTO contacts (name, addr, origin) VALUES(?, ?, ?);",
+            params![name.as_ref(), addr, origin,],
+        )
+        .is_ok()
+        {
+            row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
+            sth_modified = Modifier::Created;
         } else {
-            if origin == Origin::IncomingUnknownFrom {
-                update_authname = true;
-            }
-
-            if sql::execute(
-                context,
-                &context.sql,
-                "INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
-                params![
-                    name.as_ref(),
-                    addr,
-                    origin,
-                    if update_authname { name.as_ref() } else { "" }
-                ],
-            )
-            .is_ok()
-            {
-                row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
-                sth_modified = Modifier::Created;
-                info!(context, "added contact id={} addr={}", row_id, addr);
-            } else {
-                error!(context, "Cannot add contact.");
-            }
+            error!(context, "Cannot add contact.");
         }
 
         Ok((row_id, sth_modified))
@@ -492,18 +435,9 @@ impl Contact {
 
         for (name, addr) in split_address_book(addr_book.as_ref()).into_iter() {
             let name = normalize_name(name);
-            match Contact::add_or_lookup(context, name, addr, Origin::AddressBook) {
-                Err(err) => {
-                    warn!(
-                        context,
-                        "Failed to add address {} from address book: {}", addr, err
-                    );
-                }
-                Ok((_, modified)) => {
-                    if modified != Modifier::None {
-                        modify_cnt += 1
-                    }
-                }
+            let (_, modified) = Contact::add_or_lookup(context, name, addr, Origin::AdressBook)?;
+            if modified != Modifier::None {
+                modify_cnt += 1
             }
         }
         if modify_cnt > 0 {
@@ -557,7 +491,7 @@ impl Contact {
                 params![
                     self_addr,
                     DC_CONTACT_ID_LAST_SPECIAL as i32,
-                    Origin::IncomingReplyTo,
+                    0x100,
                     &s3str_like_cmd,
                     &s3str_like_cmd,
                     if flag_verified_only { 0 } else { 1 },
@@ -646,6 +580,8 @@ impl Contact {
             let peerstate = Peerstate::from_addr(context, &context.sql, &contact.addr);
             let loginparam = LoginParam::from_database(context, "configured_");
 
+            let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
+
             if peerstate.is_some()
                 && peerstate
                     .as_ref()
@@ -660,11 +596,16 @@ impl Contact {
                         StockMessage::E2eAvailable
                     });
                 ret += &p;
-                let self_key = Key::from(SignedPublicKey::load_self(context)?);
+                if self_key.is_none() {
+                    e2ee::ensure_secret_key_exists(context)?;
+                    self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
+                }
                 let p = context.stock_str(StockMessage::FingerPrints);
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key.formatted_fingerprint();
+                let fingerprint_self = self_key
+                    .map(|k| k.formatted_fingerprint())
+                    .unwrap_or_default();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
                     .map(|k| k.formatted_fingerprint())
@@ -768,16 +709,6 @@ impl Contact {
         Ok(Contact::load_from_db(context, contact_id)?)
     }
 
-    pub fn update_param(&mut self, context: &Context) -> Result<()> {
-        sql::execute(
-            context,
-            &context.sql,
-            "UPDATE contacts SET param=? WHERE id=?",
-            params![self.param.to_string(), self.id as i32],
-        )?;
-        Ok(())
-    }
-
     /// Get the ID of the contact.
     pub fn get_id(&self) -> u32 {
         self.id
@@ -811,9 +742,6 @@ impl Contact {
         if !self.name.is_empty() {
             return &self.name;
         }
-        if !self.authname.is_empty() {
-            return &self.authname;
-        }
         &self.addr
     }
 
@@ -849,11 +777,8 @@ impl Contact {
             if let Some(p) = context.get_config(Config::Selfavatar) {
                 return Some(PathBuf::from(p));
             }
-        } else if let Some(image_rel) = self.param.get(Param::ProfileImage) {
-            if !image_rel.is_empty() {
-                return Some(dc_get_abs_path(context, image_rel));
-            }
         }
+        // TODO: else get image_abs from contact param
         None
     }
 
@@ -936,6 +861,22 @@ impl Contact {
             .unwrap_or_default() as usize
     }
 
+    pub fn get_origin_by_id(context: &Context, contact_id: u32, ret_blocked: &mut i32) -> Origin {
+        let mut ret = Origin::Unknown;
+        *ret_blocked = 0;
+
+        if let Ok(contact) = Contact::load_from_db(context, contact_id) {
+            /* we could optimize this by loading only the needed fields */
+            if contact.blocked {
+                *ret_blocked = 1;
+            } else {
+                ret = contact.origin;
+            }
+        }
+
+        ret
+    }
+
     pub fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
         if !context.sql.is_open() || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
             return false;
@@ -1016,31 +957,6 @@ fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
     }
 }
 
-pub(crate) fn set_profile_image(
-    context: &Context,
-    contact_id: u32,
-    profile_image: &AvatarAction,
-) -> Result<()> {
-    // the given profile image is expected to be already in the blob directory
-    // as profile images can be set only by receiving messages, this should be always the case, however.
-    let mut contact = Contact::load_from_db(context, contact_id)?;
-    let changed = match profile_image {
-        AvatarAction::Change(profile_image) => {
-            contact.param.set(Param::ProfileImage, profile_image);
-            true
-        }
-        AvatarAction::Delete => {
-            contact.param.remove(Param::ProfileImage);
-            true
-        }
-    };
-    if changed {
-        contact.update_param(context)?;
-        context.call_cb(Event::ContactsChanged(Some(contact_id)));
-    }
-    Ok(())
-}
-
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
@@ -1055,7 +971,7 @@ pub fn normalize_name(full_name: impl AsRef<str>) -> String {
     }
 
     let len = full_name.len();
-    if len > 1 {
+    if len > 0 {
         let firstchar = full_name.as_bytes()[0];
         let lastchar = full_name.as_bytes()[len - 1];
         if firstchar == b'\'' && lastchar == b'\''
@@ -1105,17 +1021,6 @@ fn cat_fingerprint(
     }
 }
 
-impl Context {
-    /// determine whether the specified addr maps to the/a self addr
-    pub fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        let self_addr = self
-            .get_config(Config::ConfiguredAddr)
-            .ok_or_else(|| format_err!("Not configured"))?;
-
-        Ok(addr_cmp(self_addr, addr))
-    }
-}
-
 pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
     let norm1 = addr_normalize(addr1.as_ref()).to_lowercase();
     let norm2 = addr_normalize(addr2.as_ref()).to_lowercase();
@@ -1123,6 +1028,15 @@ pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
     norm1 == norm2
 }
 
+pub fn addr_equals_self(context: &Context, addr: impl AsRef<str>) -> bool {
+    if !addr.as_ref().is_empty() {
+        if let Some(self_addr) = context.get_config(Config::ConfiguredAddr) {
+            return addr_cmp(addr, self_addr);
+        }
+    }
+    false
+}
+
 fn split_address_book(book: &str) -> Vec<(&str, &str)> {
     book.lines()
         .chunks(2)
@@ -1163,10 +1077,6 @@ mod tests {
     fn test_normalize_name() {
         assert_eq!(&normalize_name("Doe, John"), "John Doe");
         assert_eq!(&normalize_name(" hello world   "), "hello world");
-        assert_eq!(&normalize_name("<"), "<");
-        assert_eq!(&normalize_name(">"), ">");
-        assert_eq!(&normalize_name("'"), "'");
-        assert_eq!(&normalize_name("\""), "\"");
     }
 
     #[test]
@@ -1210,18 +1120,6 @@ mod tests {
         assert_eq!(contacts.len(), 0);
     }
 
-    #[test]
-    fn test_is_self_addr() -> Result<()> {
-        let t = test_context(None);
-        assert!(t.ctx.is_self_addr("me@me.org").is_err());
-
-        let addr = configure_alice_keypair(&t.ctx);
-        assert_eq!(t.ctx.is_self_addr("me@me.org")?, false);
-        assert_eq!(t.ctx.is_self_addr(&addr)?, true);
-
-        Ok(())
-    }
-
     #[test]
     fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
@@ -1229,7 +1127,6 @@ mod tests {
         let book = concat!(
             "  Name one  \n one@eins.org \n",
             "Name two\ntwo@deux.net\n",
-            "Invalid\n+1234567890\n", // invalid, should be ignored
             "\nthree@drei.sam\n",
             "Name two\ntwo@deux.net\n" // should not be added again
         );
@@ -1312,139 +1209,6 @@ mod tests {
         assert!(!contact.is_blocked());
     }
 
-    #[test]
-    fn test_remote_authnames() {
-        let t = dummy_context();
-
-        // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
-        let (contact_id, sth_modified) = Contact::add_or_lookup(
-            &t.ctx,
-            "bob1",
-            "bob@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
-        assert_eq!(sth_modified, Modifier::Created);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "bob1");
-        assert_eq!(contact.get_name(), "bob1");
-        assert_eq!(contact.get_display_name(), "bob1");
-
-        // incoming mail `From: bob2 <bob@example.org>` - this should update authname and name
-        let (contact_id, sth_modified) = Contact::add_or_lookup(
-            &t.ctx,
-            "bob2",
-            "bob@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
-        assert_eq!(sth_modified, Modifier::Modified);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "bob2");
-        assert_eq!(contact.get_name(), "bob2");
-        assert_eq!(contact.get_display_name(), "bob2");
-
-        // manually edit name to "bob3" - authname should be still be "bob2" a given in `From:` above
-        let contact_id = Contact::create(&t.ctx, "bob3", "bob@example.org").unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "bob2");
-        assert_eq!(contact.get_name(), "bob3");
-        assert_eq!(contact.get_display_name(), "bob3");
-
-        // incoming mail `From: bob4 <bob@example.org>` - this should update authname, manually given name is still "bob3"
-        let (contact_id, sth_modified) = Contact::add_or_lookup(
-            &t.ctx,
-            "bob4",
-            "bob@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
-        assert_eq!(sth_modified, Modifier::Modified);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "bob4");
-        assert_eq!(contact.get_name(), "bob3");
-        assert_eq!(contact.get_display_name(), "bob3");
-    }
-
-    #[test]
-    fn test_remote_authnames_create_empty() {
-        let t = dummy_context();
-
-        // manually create "claire@example.org" without a given name
-        let contact_id = Contact::create(&t.ctx, "", "claire@example.org").unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "");
-        assert_eq!(contact.get_name(), "");
-        assert_eq!(contact.get_display_name(), "claire@example.org");
-
-        // incoming mail `From: claire1 <claire@example.org>` - this should update authname and name
-        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
-            &t.ctx,
-            "claire1",
-            "claire@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        assert_eq!(contact_id, contact_id_same);
-        assert_eq!(sth_modified, Modifier::Modified);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "claire1");
-        assert_eq!(contact.get_name(), "claire1");
-        assert_eq!(contact.get_display_name(), "claire1");
-
-        // incoming mail `From: claire2 <claire@example.org>` - this should update authname and name
-        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
-            &t.ctx,
-            "claire2",
-            "claire@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        assert_eq!(contact_id, contact_id_same);
-        assert_eq!(sth_modified, Modifier::Modified);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "claire2");
-        assert_eq!(contact.get_name(), "claire2");
-        assert_eq!(contact.get_display_name(), "claire2");
-    }
-
-    #[test]
-    fn test_remote_authnames_edit_empty() {
-        let t = dummy_context();
-
-        // manually create "dave@example.org"
-        let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org").unwrap();
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "");
-        assert_eq!(contact.get_name(), "dave1");
-        assert_eq!(contact.get_display_name(), "dave1");
-
-        // incoming mail `From: dave2 <dave@example.org>` - this should update authname
-        Contact::add_or_lookup(
-            &t.ctx,
-            "dave2",
-            "dave@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "dave2");
-        assert_eq!(contact.get_name(), "dave1");
-        assert_eq!(contact.get_display_name(), "dave1");
-
-        // manually clear the name
-        Contact::create(&t.ctx, "", "dave@example.org").unwrap();
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "dave2");
-        assert_eq!(contact.get_name(), "dave2");
-        assert_eq!(contact.get_display_name(), "dave2");
-    }
-
     #[test]
     fn test_addr_cmp() {
         assert!(addr_cmp("AA@AA.ORG", "aa@aa.ORG"));

src/context.rs

@@ -1,28 +1,28 @@
-//! Context module
+//! Contacts module
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::ffi::OsString;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Condvar, Mutex, RwLock};
 
+use libc::uintptr_t;
+
 use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
-use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::Event;
 use crate::imap::*;
 use crate::job::*;
 use crate::job_thread::JobThread;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message, MessengerMessage, MsgId};
+use crate::message::{self, Message, MsgId};
 use crate::param::Params;
 use crate::smtp::Smtp;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
 /// Callback function type for [Context]
 ///
@@ -32,7 +32,12 @@ use std::time::SystemTime;
 /// * `event` - One of the [Event] items.
 /// * `data1` - Depends on the event parameter, see [Event].
 /// * `data2` - Depends on the event parameter, see [Event].
-pub type ContextCallback = dyn Fn(&Context, Event) -> () + Send + Sync;
+///
+/// # Returns
+///
+/// This callback must return 0 unless stated otherwise in the event
+/// description at [Event].
+pub type ContextCallback = dyn Fn(&Context, Event) -> uintptr_t + Send + Sync;
 
 #[derive(DebugStub)]
 pub struct Context {
@@ -52,14 +57,13 @@ pub struct Context {
     #[debug_stub = "Callback"]
     cb: Box<ContextCallback>,
     pub os_name: Option<String>,
-    pub cmdline_sel_chat_id: Arc<RwLock<ChatId>>,
-    pub(crate) bob: Arc<RwLock<BobStatus>>,
+    pub cmdline_sel_chat_id: Arc<RwLock<u32>>,
+    pub bob: Arc<RwLock<BobStatus>>,
     pub last_smeared_timestamp: RwLock<i64>,
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
     pub translated_stockstrings: RwLock<HashMap<usize, String>>,
-    creation_time: SystemTime,
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -74,11 +78,20 @@ pub struct RunningState {
 /// actual keys and their values which will be present are not
 /// guaranteed.  Calling [Context::get_info] also includes information
 /// about the context on top of the information here.
-pub fn get_info() -> BTreeMap<&'static str, String> {
-    let mut res = BTreeMap::new();
+pub fn get_info() -> HashMap<&'static str, String> {
+    let mut res = HashMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
-    res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
+    res.insert(
+        "sqlite_thread_safe",
+        unsafe { rusqlite::ffi::sqlite3_threadsafe() }.to_string(),
+    );
+    res.insert(
+        "arch",
+        (std::mem::size_of::<*mut libc::c_void>())
+            .wrapping_mul(8)
+            .to_string(),
+    );
     res.insert("level", "awesome".into());
     res
 }
@@ -86,8 +99,6 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 impl Context {
     /// Creates new context.
     pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
-        pretty_env_logger::try_init_timed().ok();
-
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
@@ -121,7 +132,7 @@ impl Context {
             oauth2_critical: Arc::new(Mutex::new(())),
             bob: Arc::new(RwLock::new(Default::default())),
             last_smeared_timestamp: RwLock::new(0),
-            cmdline_sel_chat_id: Arc::new(RwLock::new(ChatId::new(0))),
+            cmdline_sel_chat_id: Arc::new(RwLock::new(0)),
             inbox_thread: Arc::new(RwLock::new(JobThread::new(
                 "INBOX",
                 "configured_inbox_folder",
@@ -141,7 +152,6 @@ impl Context {
             perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
-            creation_time: std::time::SystemTime::now(),
         };
 
         ensure!(
@@ -162,8 +172,8 @@ impl Context {
         self.blobdir.as_path()
     }
 
-    pub fn call_cb(&self, event: Event) {
-        (*self.cb)(self, event);
+    pub fn call_cb(&self, event: Event) -> uintptr_t {
+        (*self.cb)(self, event)
     }
 
     /*******************************************************************************
@@ -226,7 +236,7 @@ impl Context {
      * UI chat/message related API
      ******************************************************************************/
 
-    pub fn get_info(&self) -> BTreeMap<&'static str, String> {
+    pub fn get_info(&self) -> HashMap<&'static str, String> {
         let unset = "0";
         let l = LoginParam::from_database(self, "");
         let l2 = LoginParam::from_database(self, "configured_");
@@ -240,10 +250,7 @@ impl Context {
             .sql
             .get_raw_config_int(self, "dbversion")
             .unwrap_or_default();
-        let journal_mode = self
-            .sql
-            .query_get_value(self, "PRAGMA journal_mode;", rusqlite::NO_PARAMS)
-            .unwrap_or_else(|| "unknown".to_string());
+
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
         let bcc_self = self.get_config_int(Config::BccSelf);
@@ -258,9 +265,10 @@ impl Context {
             rusqlite::NO_PARAMS,
         );
 
-        let fingerprint_str = match SignedPublicKey::load_self(self) {
-            Ok(key) => Key::from(key).fingerprint(),
-            Err(err) => format!("<key failure: {}>", err),
+        let fingerprint_str = if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql) {
+            key.fingerprint()
+        } else {
+            "<Not yet calculated>".into()
         };
 
         let inbox_watch = self.get_config_int(Config::InboxWatch);
@@ -288,14 +296,8 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
-        res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
-        res.insert(
-            "selfavatar",
-            self.get_config(Config::Selfavatar)
-                .unwrap_or_else(|| "<unset>".to_string()),
-        );
         res.insert("is_configured", is_configured.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
@@ -319,9 +321,6 @@ impl Context {
         );
         res.insert("fingerprint", fingerprint_str);
 
-        let elapsed = self.creation_time.elapsed();
-        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
-
         res
     }
 
@@ -357,7 +356,7 @@ impl Context {
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -365,7 +364,7 @@ impl Context {
         let strLikeInText = format!("%{}%", real_query);
         let strLikeBeg = format!("{}%", real_query);
 
-        let query = if !chat_id.is_unset() {
+        let query = if 0 != chat_id {
             concat!(
                 "SELECT m.id AS id, m.timestamp AS timestamp",
                 " FROM msgs m",
@@ -397,7 +396,7 @@ impl Context {
         self.sql
             .query_map(
                 query,
-                params![chat_id, &strLikeInText, &strLikeBeg],
+                params![chat_id as i32, &strLikeInText, &strLikeBeg],
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -448,17 +447,15 @@ impl Context {
                 return;
             }
 
-            match msg.is_dc_message {
-                MessengerMessage::No => {}
-                MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job_add(
-                        self,
-                        Action::MoveMsg,
-                        msg.id.to_u32() as i32,
-                        Params::new(),
-                        0,
-                    );
-                }
+            // 1 = dc message, 2 = reply to dc message
+            if 0 != msg.is_dc_message {
+                job_add(
+                    self,
+                    Action::MoveMsg,
+                    msg.id.to_u32() as i32,
+                    Params::new(),
+                    0,
+                );
             }
         }
     }
@@ -488,14 +485,14 @@ impl Default for RunningState {
 }
 
 #[derive(Debug, Default)]
-pub(crate) struct BobStatus {
+pub struct BobStatus {
     pub expects: i32,
     pub status: i32,
     pub qr_scan: Option<Lot>,
 }
 
 #[derive(Debug, PartialEq)]
-pub(crate) enum PerformJobsNeeded {
+pub enum PerformJobsNeeded {
     Not,
     AtOnce,
     AvoidDos,
@@ -512,7 +509,7 @@ pub struct SmtpState {
     pub idle: bool,
     pub suspended: bool,
     pub doing_jobs: bool,
-    pub(crate) perform_jobs_needed: PerformJobsNeeded,
+    pub perform_jobs_needed: PerformJobsNeeded,
     pub probe_network: bool,
 }
 
@@ -531,7 +528,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| ()), "FakeOs".into(), dbfile);
+        let res = Context::new(Box::new(|_, _| 0), "FakeOs".into(), dbfile);
         assert!(res.is_err());
     }
 
@@ -546,7 +543,7 @@ mod tests {
     fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
+        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -557,7 +554,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile);
+        let res = Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile);
         assert!(res.is_err());
     }
 
@@ -567,7 +564,7 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
+        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -577,7 +574,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 
@@ -586,7 +583,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 

src/dc_array.rs

@@ -2,6 +2,7 @@ use crate::location::Location;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
+#[allow(non_camel_case_types)]
 pub enum dc_array_t {
     Locations(Vec<Location>),
     Uint(Vec<u32>),

src/dc_simplify.rs

@@ -0,0 +1,273 @@
+use crate::dehtml::*;
+
+#[derive(Copy, Clone)]
+pub struct Simplify {
+    pub is_forwarded: bool,
+}
+
+/// Return index of footer line in vector of message lines, or vector length if
+/// no footer is found.
+///
+/// Also return whether not-standard (rfc3676, §4.3) footer is found.
+fn find_message_footer(lines: &[&str]) -> (usize, bool) {
+    for (ix, &line) in lines.iter().enumerate() {
+        // quoted-printable may encode `-- ` to `-- =20` which is converted
+        // back to `--  `
+        match line {
+            "-- " | "--  " => return (ix, false),
+            "--" | "---" | "----" => return (ix, true),
+            _ => (),
+        }
+    }
+    (lines.len(), false)
+}
+
+impl Simplify {
+    pub fn new() -> Self {
+        Simplify {
+            is_forwarded: false,
+        }
+    }
+
+    /// Simplify and normalise text: Remove quotes, signatures, unnecessary
+    /// lineends etc.
+    /// The data returned from simplify() must be free()'d when no longer used.
+    pub fn simplify(&mut self, input: &str, is_html: bool, is_msgrmsg: bool) -> String {
+        let mut out = if is_html {
+            dehtml(input)
+        } else {
+            input.to_string()
+        };
+
+        out.retain(|c| c != '\r');
+        out = self.simplify_plain_text(&out, is_msgrmsg);
+        out.retain(|c| c != '\r');
+
+        out
+    }
+
+    /**
+     * Simplify Plain Text
+     */
+    #[allow(non_snake_case, clippy::mut_range_bound)]
+    fn simplify_plain_text(&mut self, buf_terminated: &str, is_msgrmsg: bool) -> String {
+        /* This function ...
+        ... removes all text after the line `-- ` (footer mark)
+        ... removes full quotes at the beginning and at the end of the text -
+            these are all lines starting with the character `>`
+        ... remove a non-empty line before the removed quote (contains sth. like "On 2.9.2016, Bjoern wrote:" in different formats and lanugages) */
+        /* split the given buffer into lines */
+        let lines: Vec<_> = buf_terminated.split('\n').collect();
+        let mut l_first: usize = 0;
+        let mut is_cut_at_begin = false;
+        let (mut l_last, mut is_cut_at_end) = find_message_footer(&lines);
+
+        if l_last > l_first + 2 {
+            let line0 = lines[l_first];
+            let line1 = lines[l_first + 1];
+            let line2 = lines[l_first + 2];
+            if line0 == "---------- Forwarded message ----------"
+                && line1.starts_with("From: ")
+                && line2.is_empty()
+            {
+                self.is_forwarded = true;
+                l_first += 3
+            }
+        }
+        for l in l_first..l_last {
+            let line = lines[l];
+            if line == "-----"
+                || line == "_____"
+                || line == "====="
+                || line == "*****"
+                || line == "~~~~~"
+            {
+                l_last = l;
+                is_cut_at_end = true;
+                /* done */
+                break;
+            }
+        }
+        if !is_msgrmsg {
+            let mut l_lastQuotedLine = None;
+            for l in (l_first..l_last).rev() {
+                let line = lines[l];
+                if is_plain_quote(line) {
+                    l_lastQuotedLine = Some(l)
+                } else if !is_empty_line(line) {
+                    break;
+                }
+            }
+            if let Some(last_quoted_line) = l_lastQuotedLine {
+                l_last = last_quoted_line;
+                is_cut_at_end = true;
+                if l_last > 1 && is_empty_line(lines[l_last - 1]) {
+                    l_last -= 1
+                }
+                if l_last > 1 {
+                    let line = lines[l_last - 1];
+                    if is_quoted_headline(line) {
+                        l_last -= 1
+                    }
+                }
+            }
+        }
+        if !is_msgrmsg {
+            let mut l_lastQuotedLine_0 = None;
+            let mut hasQuotedHeadline = 0;
+            for l in l_first..l_last {
+                let line = lines[l];
+                if is_plain_quote(line) {
+                    l_lastQuotedLine_0 = Some(l)
+                } else if !is_empty_line(line) {
+                    if is_quoted_headline(line)
+                        && 0 == hasQuotedHeadline
+                        && l_lastQuotedLine_0.is_none()
+                    {
+                        hasQuotedHeadline = 1i32
+                    } else {
+                        /* non-quoting line found */
+                        break;
+                    }
+                }
+            }
+            if let Some(last_quoted_line) = l_lastQuotedLine_0 {
+                l_first = last_quoted_line + 1;
+                is_cut_at_begin = true
+            }
+        }
+        /* re-create buffer from the remaining lines */
+        let mut ret = String::new();
+        if is_cut_at_begin {
+            ret += "[...]";
+        }
+        /* we write empty lines only in case and non-empty line follows */
+        let mut pending_linebreaks = 0;
+        let mut content_lines_added = 0;
+        for l in l_first..l_last {
+            let line = lines[l];
+            if is_empty_line(line) {
+                pending_linebreaks += 1
+            } else {
+                if 0 != content_lines_added {
+                    if pending_linebreaks > 2i32 {
+                        pending_linebreaks = 2i32
+                    }
+                    while 0 != pending_linebreaks {
+                        ret += "\n";
+                        pending_linebreaks -= 1
+                    }
+                }
+                // the incoming message might contain invalid UTF8
+                ret += line;
+                content_lines_added += 1;
+                pending_linebreaks = 1i32
+            }
+        }
+        if is_cut_at_end && (!is_cut_at_begin || 0 != content_lines_added) {
+            ret += " [...]";
+        }
+
+        ret
+    }
+}
+
+/**
+ * Tools
+ */
+fn is_empty_line(buf: &str) -> bool {
+    // XXX: can it be simplified to buf.chars().all(|c| c.is_whitespace())?
+    //
+    // Strictly speaking, it is not equivalent (^A is not whitespace, but less than ' '),
+    // but having control sequences in email body?!
+    //
+    // See discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
+    for c in buf.chars() {
+        if c > ' ' {
+            return false;
+        }
+    }
+
+    true
+}
+
+fn is_quoted_headline(buf: &str) -> bool {
+    /* This function may be called for the line _directly_ before a quote.
+    The function checks if the line contains sth. like "On 01.02.2016, xy@z wrote:" in various languages.
+    - Currently, we simply check if the last character is a ':'.
+    - Checking for the existence of an email address may fail (headlines may show the user's name instead of the address) */
+
+    buf.len() <= 80 && buf.ends_with(':')
+}
+
+fn is_plain_quote(buf: &str) -> bool {
+    buf.starts_with('>')
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::prelude::*;
+
+    proptest! {
+        #[test]
+        // proptest does not support [[:graphical:][:space:]] regex.
+        fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
+            let output = Simplify::new().simplify_plain_text(&input, true);
+            assert!(output.split('\n').all(|s| s != "-- "));
+        }
+    }
+
+    #[test]
+    fn test_simplify_trim() {
+        let mut simplify = Simplify::new();
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
+        let plain = simplify.simplify(html, true, false);
+
+        assert_eq!(plain, "line1\nline2");
+    }
+
+    #[test]
+    fn test_simplify_parse_href() {
+        let mut simplify = Simplify::new();
+        let html = "<a href=url>text</a";
+        let plain = simplify.simplify(html, true, false);
+
+        assert_eq!(plain, "[text](url)");
+    }
+
+    #[test]
+    fn test_simplify_bold_text() {
+        let mut simplify = Simplify::new();
+        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
+        let plain = simplify.simplify(html, true, false);
+
+        assert_eq!(plain, "text *bold*<>");
+    }
+
+    #[test]
+    fn test_simplify_html_encoded() {
+        let mut simplify = Simplify::new();
+        let html =
+                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
+
+        let plain = simplify.simplify(html, true, false);
+
+        assert_eq!(
+            plain,
+            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
+        );
+    }
+
+    #[test]
+    fn test_simplify_utilities() {
+        assert!(is_empty_line(" \t"));
+        assert!(is_empty_line(""));
+        assert!(is_empty_line(" \r"));
+        assert!(!is_empty_line(" x"));
+        assert!(is_plain_quote("> hello world"));
+        assert!(is_plain_quote(">>"));
+        assert!(!is_plain_quote("Life is pain"));
+        assert!(!is_plain_quote(""));
+    }
+}

src/dc_tools.rs

@@ -5,24 +5,24 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::time::{Duration, SystemTime};
+use std::time::SystemTime;
 use std::{fmt, fs};
 
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::{bail, Error};
+use crate::error::Error;
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
     0 != v && 0 == v & (v - 1)
 }
 
-/// Shortens a string to a specified length and adds "[...]" to the
-/// end of the shortened string.
-pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
-    let ellipse = "[...]";
+/// Shortens a string to a specified length and adds "..." or "[...]" to the end of
+/// the shortened string.
+pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Cow<str> {
+    let ellipse = if do_unwrap { "..." } else { "[...]" };
 
     let count = buf.chars().count();
     if approx_chars > 0 && count > approx_chars + ellipse.len() {
@@ -49,8 +49,8 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
 /// - harmonize together while being different enough
 /// (therefore, we cannot just use random rgb colors :)
 const COLORS: [u32; 16] = [
-    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
-    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
+    0xe56555, 0xf28c48, 0x8e85ee, 0x76c84d, 0x5bb6cc, 0x549cdd, 0xd25c99, 0xb37800, 0xf23030,
+    0x39b249, 0xbb243b, 0x964078, 0x66874f, 0x308ab9, 0x127ed0, 0xbe450c,
 ];
 
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
@@ -59,7 +59,7 @@ pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let bytes = str_lower.as_bytes();
     for (i, byte) in bytes.iter().enumerate() {
         checksum += (i + 1) * *byte as usize;
-        checksum %= 0x00ff_ffff;
+        checksum %= 0xffffff;
     }
     let color_index = checksum % COLORS.len();
 
@@ -75,14 +75,6 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String {
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
-pub fn duration_to_str(duration: Duration) -> String {
-    let secs = duration.as_secs();
-    let h = secs / 3600;
-    let m = (secs % 3600) / 60;
-    let s = (secs % 3600) % 60;
-    format!("{}h {}m {}s", h, m, s)
-}
-
 pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
@@ -190,6 +182,22 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
     String::from_utf8(wrapped_writer).unwrap()
 }
 
+pub(crate) fn dc_create_incoming_rfc724_mid(
+    message_timestamp: i64,
+    contact_id_from: u32,
+    contact_ids_to: &[u32],
+) -> Option<String> {
+    /* create a deterministic rfc724_mid from input such that
+    repeatedly calling it with the same input results in the same Message-id */
+
+    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
+    let result = format!(
+        "{}-{}-{}@stub",
+        message_timestamp, contact_id_from, largest_id_to
+    );
+    Some(result)
+}
+
 /// Function generates a Message-ID that can be used for a new outgoing message.
 /// - this function is called for all outgoing messages.
 /// - the message ID should be globally unique
@@ -230,7 +238,45 @@ pub(crate) fn dc_extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> {
     None
 }
 
+// Function returns a sanitized basename that does not contain
+// win/linux path separators and also not any non-ascii chars
+fn get_safe_basename(filename: &str) -> String {
+    // return the (potentially mangled) basename of the input filename
+    // this might be a path that comes in from another operating system
+    let mut index: usize = 0;
+
+    if let Some(unix_index) = filename.rfind('/') {
+        index = unix_index + 1;
+    }
+    if let Some(win_index) = filename.rfind('\\') {
+        index = max(index, win_index + 1);
+    }
+    if index >= filename.len() {
+        "nobasename".to_string()
+    } else {
+        // we don't allow any non-ascii to be super-safe
+        filename[index..].replace(|c: char| !c.is_ascii() || c == ':', "-")
+    }
+}
+
+pub fn dc_derive_safe_stem_ext(filename: &str) -> (String, String) {
+    let basename = get_safe_basename(&filename);
+    let (mut stem, mut ext) = if let Some(index) = basename.rfind('.') {
+        (
+            basename[0..index].to_string(),
+            basename[index..].to_string(),
+        )
+    } else {
+        (basename, "".to_string())
+    };
+    // limit length of stem and ext
+    stem.truncate(32);
+    ext.truncate(32);
+    (stem, ext)
+}
+
 // the returned suffix is lower-case
+#[allow(non_snake_case)]
 pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
     Path::new(path_filename.as_ref())
         .extension()
@@ -351,14 +397,11 @@ pub(crate) fn dc_copy_file(
     }
 }
 
-pub(crate) fn dc_create_folder(
-    context: &Context,
-    path: impl AsRef<std::path::Path>,
-) -> Result<(), std::io::Error> {
+pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs) {
-            Ok(_) => Ok(()),
+            Ok(_) => true,
             Err(err) => {
                 warn!(
                     context,
@@ -366,11 +409,11 @@ pub(crate) fn dc_create_folder(
                     path.as_ref().display(),
                     err
                 );
-                Err(err)
+                false
             }
         }
     } else {
-        Ok(())
+        true
     }
 }
 
@@ -456,27 +499,10 @@ pub(crate) fn dc_get_next_backup_path(
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap_or_default()
+        .unwrap()
         .as_secs() as i64
 }
 
-/// An invalid email address was encountered
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid email address: {message} ({addr})")]
-pub struct InvalidEmailError {
-    message: String,
-    addr: String,
-}
-
-impl InvalidEmailError {
-    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
-        InvalidEmailError {
-            message: msg.into(),
-            addr: addr.into(),
-        }
-    }
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -493,14 +519,14 @@ impl InvalidEmailError {
 /// assert_eq!(&email.domain, "example.com");
 /// assert_eq!(email.to_string(), "someone@example.com");
 /// ```
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct EmailAddress {
     pub local: String,
     pub domain: String,
 }
 
 impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, InvalidEmailError> {
+    pub fn new(input: &str) -> Result<Self, Error> {
         input.parse::<EmailAddress>()
     }
 }
@@ -512,55 +538,31 @@ impl fmt::Display for EmailAddress {
 }
 
 impl FromStr for EmailAddress {
-    type Err = InvalidEmailError;
+    type Err = Error;
 
     /// Performs a dead-simple parse of an email address.
-    fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        if input.is_empty() {
-            return Err(InvalidEmailError::new("empty string is not valid", input));
-        }
+    fn from_str(input: &str) -> Result<EmailAddress, Error> {
+        ensure!(!input.is_empty(), "empty string is not valid");
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
-        let err = |msg: &str| {
-            Err(InvalidEmailError {
-                message: msg.to_string(),
-                addr: input.to_string(),
-            })
-        };
-        match &parts[..] {
-            [domain, local] => {
-                if local.is_empty() {
-                    return err("empty string is not valid for local part");
-                }
-                if domain.len() <= 3 {
-                    return err("domain is too short");
-                }
-                let dot = domain.find('.');
-                match dot {
-                    None => {
-                        return err("invalid domain");
-                    }
-                    Some(dot_idx) => {
-                        if dot_idx >= domain.len() - 2 {
-                            return err("invalid domain");
-                        }
-                    }
-                }
-                Ok(EmailAddress {
-                    local: (*local).to_string(),
-                    domain: (*domain).to_string(),
-                })
-            }
-            _ => err("missing '@' character"),
-        }
-    }
-}
+        ensure!(parts.len() > 1, "missing '@' character");
+        let local = parts[1];
+        let domain = parts[0];
 
-impl rusqlite::types::ToSql for EmailAddress {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-        let val = rusqlite::types::Value::Text(self.to_string());
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
+        ensure!(
+            !local.is_empty(),
+            "empty string is not valid for local part"
+        );
+        ensure!(domain.len() > 3, "domain is too short");
+
+        let dot = domain.find('.');
+        ensure!(dot.is_some(), "invalid domain");
+        ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
+
+        Ok(EmailAddress {
+            local: local.to_string(),
+            domain: domain.to_string(),
+        })
     }
 }
 
@@ -587,42 +589,54 @@ mod tests {
     #[test]
     fn test_dc_truncate_1() {
         let s = "this is a little test string";
-        assert_eq!(dc_truncate(s, 16), "this is a [...]");
+        assert_eq!(dc_truncate(s, 16, false), "this is a [...]");
+        assert_eq!(dc_truncate(s, 16, true), "this is a ...");
     }
 
     #[test]
     fn test_dc_truncate_2() {
-        assert_eq!(dc_truncate("1234", 2), "1234");
+        assert_eq!(dc_truncate("1234", 2, false), "1234");
+        assert_eq!(dc_truncate("1234", 2, true), "1234");
     }
 
     #[test]
     fn test_dc_truncate_3() {
-        assert_eq!(dc_truncate("1234567", 1), "1[...]");
+        assert_eq!(dc_truncate("1234567", 1, false), "1[...]");
+        assert_eq!(dc_truncate("1234567", 1, true), "1...");
     }
 
     #[test]
     fn test_dc_truncate_4() {
-        assert_eq!(dc_truncate("123456", 4), "123456");
+        assert_eq!(dc_truncate("123456", 4, false), "123456");
+        assert_eq!(dc_truncate("123456", 4, true), "123456");
     }
 
     #[test]
     fn test_dc_truncate_edge() {
-        assert_eq!(dc_truncate("", 4), "");
+        assert_eq!(dc_truncate("", 4, false), "");
+        assert_eq!(dc_truncate("", 4, true), "");
 
-        assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
+        assert_eq!(dc_truncate("\n  hello \n world", 4, false), "\n  [...]");
+        assert_eq!(dc_truncate("\n  hello \n world", 4, true), "\n  ...");
 
-        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
         assert_eq!(
-            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1, false),
+            "𐠈[...]"
+        );
+        assert_eq!(
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0, false),
             "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
         );
 
         // 9 characters, so no truncation
-        assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
+        assert_eq!(
+            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6, false),
+            "𑒀ὐ￠🜀\u{1e01b}A a🟠",
+        );
 
         // 12 characters, truncation
         assert_eq!(
-            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6),
+            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6, false),
             "𑒀ὐ￠🜀\u{1e01b}A[...]",
         );
     }
@@ -653,6 +667,7 @@ mod tests {
         );
     }
 
+    #[test]
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid
@@ -707,29 +722,29 @@ mod tests {
 
     #[test]
     fn test_emailaddress_parse() {
-        assert_eq!("".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!(EmailAddress::new("").is_ok(), false);
         assert_eq!(
-            "user@domain.tld".parse::<EmailAddress>().unwrap(),
+            EmailAddress::new("user@domain.tld").unwrap(),
             EmailAddress {
                 local: "user".into(),
                 domain: "domain.tld".into(),
             }
         );
-        assert_eq!("uuu".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("dd.tt".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("tt.dd@uu".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d.".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d.t".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!(EmailAddress::new("uuu").is_ok(), false);
+        assert_eq!(EmailAddress::new("dd.tt").is_ok(), false);
+        assert_eq!(EmailAddress::new("tt.dd@uu").is_ok(), false);
+        assert_eq!(EmailAddress::new("u@d").is_ok(), false);
+        assert_eq!(EmailAddress::new("u@d.").is_ok(), false);
+        assert_eq!(EmailAddress::new("u@d.t").is_ok(), false);
         assert_eq!(
-            "u@d.tt".parse::<EmailAddress>().unwrap(),
+            EmailAddress::new("u@d.tt").unwrap(),
             EmailAddress {
                 local: "u".into(),
                 domain: "d.tt".into(),
             }
         );
-        assert_eq!("u@tt".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!(EmailAddress::new("u@.tt").is_ok(), false);
+        assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
     }
 
     use proptest::prelude::*;
@@ -738,10 +753,11 @@ mod tests {
         #[test]
         fn test_dc_truncate(
             buf: String,
-            approx_chars in 0..10000usize
+            approx_chars in 0..10000usize,
+            do_unwrap: bool,
         ) {
-            let res = dc_truncate(&buf, approx_chars);
-            let el_len = 5;
+            let res = dc_truncate(&buf, approx_chars, do_unwrap);
+            let el_len = if do_unwrap { 3 } else { 5 };
             let l = res.chars().count();
             if approx_chars > 0 {
                 assert!(
@@ -755,11 +771,36 @@ mod tests {
 
             if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
                 let l = res.len();
-                assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
+                if do_unwrap {
+                    assert_eq!(&res[l-3..l], "...", "missing ellipsis in {}", &res);
+                } else {
+                    assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
+                }
             }
         }
     }
 
+    #[test]
+    fn test_dc_create_incoming_rfc724_mid() {
+        let res = dc_create_incoming_rfc724_mid(123, 45, &[6, 7]);
+        assert_eq!(res, Some("123-45-7@stub".into()));
+        let res = dc_create_incoming_rfc724_mid(123, 45, &[]);
+        assert_eq!(res, Some("123-45-0@stub".into()));
+    }
+
+    #[test]
+    fn test_file_get_safe_basename() {
+        assert_eq!(get_safe_basename("12312/hello"), "hello");
+        assert_eq!(get_safe_basename("12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12/\\\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//"), "nobasename");
+        assert_eq!(get_safe_basename("//123:12/"), "nobasename");
+        assert!(get_safe_basename("123\x012.hello").ends_with(".hello"));
+    }
+
     #[test]
     fn test_file_handling() {
         let t = dummy_context();
@@ -808,7 +849,7 @@ mod tests {
 
         assert!(dc_delete_file(context, "$BLOBDIR/foobar"));
         assert!(dc_delete_file(context, "$BLOBDIR/dada"));
-        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder").is_ok());
+        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder"));
         assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
 
@@ -862,37 +903,4 @@ mod tests {
         let next = dc_smeared_time(&t.ctx);
         assert!((start + count - 1) < next);
     }
-
-    #[test]
-    fn test_duration_to_str() {
-        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
-        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
-        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 59)),
-            "0h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 60)),
-            "1h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
-            "2h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
-            "3h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
-            "3h 0m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
-            "3h 1m 0s"
-        );
-    }
 }

src/dehtml.rs

@@ -3,6 +3,7 @@
 //! A module to remove HTML tags from the email text
 
 use lazy_static::lazy_static;
+use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 lazy_static! {
@@ -34,7 +35,6 @@ pub fn dehtml(buf: &str) -> String {
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
-    reader.check_end_names(false);
 
     let mut buf = Vec::new();
 
@@ -139,12 +139,13 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
-            if let Some(href) = event
-                .html_attributes()
-                .filter_map(|attr| attr.ok())
-                .find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "href")
-            {
+            if let Some(href) = event.html_attributes().find(|attr| {
+                attr.as_ref()
+                    .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "href")
+                    .unwrap_or_default()
+            }) {
                 let href = href
+                    .unwrap()
                     .unescape_and_decode_value(reader)
                     .unwrap_or_default()
                     .to_lowercase();
@@ -188,60 +189,4 @@ mod tests {
             assert_eq!(dehtml(input), output);
         }
     }
-
-    #[test]
-    fn test_dehtml_parse_br() {
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
-        let plain = dehtml(html);
-
-        assert_eq!(plain, "line1\n\r\r\rline2");
-    }
-
-    #[test]
-    fn test_dehtml_parse_href() {
-        let html = "<a href=url>text</a";
-        let plain = dehtml(html);
-
-        assert_eq!(plain, "[text](url)");
-    }
-
-    #[test]
-    fn test_dehtml_bold_text() {
-        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = dehtml(html);
-
-        assert_eq!(plain, "text *bold*<>");
-    }
-
-    #[test]
-    fn test_dehtml_html_encoded() {
-        let html =
-                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
-
-        let plain = dehtml(html);
-
-        assert_eq!(
-            plain,
-            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
-        );
-    }
-
-    #[test]
-    fn test_unclosed_tags() {
-        let input = r##"
-        <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'
-        'http://www.w3.org/TR/html4/loose.dtd'>
-        <html>
-        <head>
-        <title>Hi</title>
-        <meta http-equiv='Content-Type' content='text/html; charset=iso-8859-1'>						
-        </head>
-        <body>
-        lots of text
-        </body>
-        </html>
-        "##;
-        let txt = dehtml(input);
-        assert_eq!(txt.trim(), "lots of text");
-    }
 }

src/e2ee.rs

@@ -2,26 +2,25 @@
 
 use std::collections::HashSet;
 
-use mailparse::ParsedMail;
+use mailparse::MailHeaderMap;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
 use crate::error::*;
-use crate::headerdef::HeaderDef;
-use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
+use crate::key::*;
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
 use crate::securejoin::handle_degrade_event;
+use crate::wrapmime;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
     pub prefer_encrypt: EncryptPreference,
     pub addr: String,
-    pub public_key: SignedPublicKey,
+    pub public_key: Key,
 }
 
 impl EncryptHelper {
@@ -36,7 +35,7 @@ impl EncryptHelper {
             Some(addr) => addr,
         };
 
-        let public_key = SignedPublicKey::load_self(context)?;
+        let public_key = load_or_generate_self_public_key(context, &addr)?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -104,9 +103,10 @@ impl EncryptHelper {
             })?;
             keyring.add_ref(key);
         }
-        let public_key = Key::from(self.public_key.clone());
-        keyring.add_ref(&public_key);
-        let sign_key = Key::from(SignedSecretKey::load_self(context)?);
+
+        keyring.add_ref(&self.public_key);
+        let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql)
+            .ok_or_else(|| format_err!("missing own private key"))?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
@@ -118,13 +118,13 @@ impl EncryptHelper {
 
 pub fn try_decrypt(
     context: &Context,
-    mail: &ParsedMail<'_>,
+    mail: &mailparse::ParsedMail<'_>,
     message_time: i64,
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
         .headers
-        .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
+        .get_first_value("From")?
+        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -186,36 +186,55 @@ pub fn try_decrypt(
     Ok((out_mail, signatures))
 }
 
-/// Returns a reference to the encrypted payload and validates the autocrypt structure.
-fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
-    ensure!(
-        mail.ctype.mimetype == "multipart/encrypted",
-        "Not a multipart/encrypted message: {}",
-        mail.ctype.mimetype
-    );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
+/// Load public key from database or generate a new one.
+///
+/// This will load a public key from the database, generating and
+/// storing a new one when one doesn't exist yet.  Care is taken to
+/// only generate one key per context even when multiple threads call
+/// this function concurrently.
+fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str>) -> Result<Key> {
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return Ok(key);
+    }
+    let _guard = context.generating_key_mutex.lock().unwrap();
 
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
+    // Check again in case the key was generated while we were waiting for the lock.
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return Ok(key);
+    }
 
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
+    let start = std::time::Instant::now();
+    info!(
+        context,
+        "Generating keypair with {} bits, e={} ...", 2048, 65537,
     );
-
-    Ok(&mail.subparts[1])
+    match pgp::create_keypair(&self_addr) {
+        Some((public_key, private_key)) => {
+            if dc_key_save_self_keypair(
+                context,
+                &public_key,
+                &private_key,
+                &self_addr,
+                true,
+                &context.sql,
+            ) {
+                info!(
+                    context,
+                    "Keypair generated in {:.3}s.",
+                    start.elapsed().as_secs()
+                );
+                Ok(public_key)
+            } else {
+                Err(format_err!("Failed to save keypair"))
+            }
+        }
+        None => Err(format_err!("Failed to generate keypair")),
+    }
 }
 
 fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
-    mail: &ParsedMail<'a>,
+    mail: &mailparse::ParsedMail<'a>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
@@ -227,7 +246,7 @@ fn decrypt_if_autocrypt_message<'a>(
     //
     // Errors are returned for failures related to decryption of AC-messages.
 
-    let encrypted_data_part = match get_autocrypt_mime(mail) {
+    let encrypted_data_part = match wrapmime::get_autocrypt_mime(mail) {
         Err(_) => {
             // not an autocrypt mime message, abort and ignore
             return Ok(None);
@@ -248,7 +267,7 @@ fn decrypt_if_autocrypt_message<'a>(
 /// Returns Ok(None) if nothing encrypted was found.
 fn decrypt_part(
     _context: &Context,
-    mail: &ParsedMail<'_>,
+    mail: &mailparse::ParsedMail<'_>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
@@ -294,7 +313,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
 /// However, Delta Chat itself has no problem with encrypted multipart/report
 /// parts and MUAs should be encouraged to encrpyt multipart/reports as well so
 /// that we could use the normal Autocrypt processing.
-fn contains_report(mail: &ParsedMail<'_>) -> bool {
+fn contains_report(mail: &mailparse::ParsedMail<'_>) -> bool {
     mail.ctype.mimetype == "multipart/report"
 }
 
@@ -307,7 +326,6 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 ///
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
-// TODO, remove this once deltachat::key::Key no longer exists.
 pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
         format_err!(concat!(
@@ -315,7 +333,7 @@ pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
             "cannot ensure secret key if not configured."
         ))
     })?;
-    SignedPublicKey::load_self(context)?;
+    load_or_generate_self_public_key(context, &self_addr)?;
     Ok(self_addr)
 }
 
@@ -366,6 +384,49 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
+    mod load_or_generate_self_public_key {
+        use super::*;
+
+        #[test]
+        fn test_existing() {
+            let t = dummy_context();
+            let addr = configure_alice_keypair(&t.ctx);
+            let key = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key.is_ok());
+        }
+
+        #[test]
+        #[ignore] // generating keys is expensive
+        fn test_generate() {
+            let t = dummy_context();
+            let addr = "alice@example.org";
+            let key0 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key0.is_ok());
+            let key1 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key1.is_ok());
+            assert_eq!(key0.unwrap(), key1.unwrap());
+        }
+
+        #[test]
+        #[ignore]
+        fn test_generate_concurrent() {
+            use std::sync::Arc;
+            use std::thread;
+
+            let t = dummy_context();
+            let ctx = Arc::new(t.ctx);
+            let ctx0 = Arc::clone(&ctx);
+            let thr0 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx0, "alice@example.org"));
+            let ctx1 = Arc::clone(&ctx);
+            let thr1 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx1, "alice@example.org"));
+            let res0 = thr0.join().unwrap();
+            let res1 = thr1.join().unwrap();
+            assert_eq!(res0.unwrap(), res1.unwrap());
+        }
+    }
+
     #[test]
     fn test_has_decrypted_pgp_armor() {
         let data = b" -----BEGIN PGP MESSAGE-----";

src/error.rs

@@ -1,13 +1,154 @@
 //! # Error handling
 
-pub use anyhow::{bail, ensure, format_err, Error, Result};
+use lettre_email::mime;
 
-// #[fail(display = "Invalid Message ID.")]
-// InvalidMsgId,
-// #[fail(display = "Watch folder not found {:?}", _0)]
-// WatchFolderNotFound(String),
-// #[fail(display = "Not Configured")]
-// NotConfigured,
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "{:?}", _0)]
+    Failure(failure::Error),
+    #[fail(display = "SQL error: {:?}", _0)]
+    SqlError(#[cause] crate::sql::Error),
+    #[fail(display = "{:?}", _0)]
+    Io(std::io::Error),
+    #[fail(display = "{:?}", _0)]
+    Message(String),
+    #[fail(display = "{:?}", _0)]
+    Image(image_meta::ImageError),
+    #[fail(display = "{:?}", _0)]
+    Utf8(std::str::Utf8Error),
+    #[fail(display = "PGP: {:?}", _0)]
+    Pgp(pgp::errors::Error),
+    #[fail(display = "Base64Decode: {:?}", _0)]
+    Base64Decode(base64::DecodeError),
+    #[fail(display = "{:?}", _0)]
+    FromUtf8(std::string::FromUtf8Error),
+    #[fail(display = "{}", _0)]
+    BlobError(#[cause] crate::blob::BlobError),
+    #[fail(display = "Invalid Message ID.")]
+    InvalidMsgId,
+    #[fail(display = "Watch folder not found {:?}", _0)]
+    WatchFolderNotFound(String),
+    #[fail(display = "Inalid Email: {:?}", _0)]
+    MailParseError(#[cause] mailparse::MailParseError),
+    #[fail(display = "Building invalid Email: {:?}", _0)]
+    LettreError(#[cause] lettre_email::error::Error),
+    #[fail(display = "FromStr error: {:?}", _0)]
+    FromStr(#[cause] mime::FromStrError),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<crate::sql::Error> for Error {
+    fn from(err: crate::sql::Error) -> Error {
+        Error::SqlError(err)
+    }
+}
+
+impl From<base64::DecodeError> for Error {
+    fn from(err: base64::DecodeError) -> Error {
+        Error::Base64Decode(err)
+    }
+}
+
+impl From<failure::Error> for Error {
+    fn from(err: failure::Error) -> Error {
+        Error::Failure(err)
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Error {
+        Error::Io(err)
+    }
+}
+
+impl From<std::str::Utf8Error> for Error {
+    fn from(err: std::str::Utf8Error) -> Error {
+        Error::Utf8(err)
+    }
+}
+
+impl From<image_meta::ImageError> for Error {
+    fn from(err: image_meta::ImageError) -> Error {
+        Error::Image(err)
+    }
+}
+
+impl From<pgp::errors::Error> for Error {
+    fn from(err: pgp::errors::Error) -> Error {
+        Error::Pgp(err)
+    }
+}
+
+impl From<std::string::FromUtf8Error> for Error {
+    fn from(err: std::string::FromUtf8Error) -> Error {
+        Error::FromUtf8(err)
+    }
+}
+
+impl From<crate::blob::BlobError> for Error {
+    fn from(err: crate::blob::BlobError) -> Error {
+        Error::BlobError(err)
+    }
+}
+
+impl From<crate::message::InvalidMsgId> for Error {
+    fn from(_err: crate::message::InvalidMsgId) -> Error {
+        Error::InvalidMsgId
+    }
+}
+
+impl From<mailparse::MailParseError> for Error {
+    fn from(err: mailparse::MailParseError) -> Error {
+        Error::MailParseError(err)
+    }
+}
+
+impl From<lettre_email::error::Error> for Error {
+    fn from(err: lettre_email::error::Error) -> Error {
+        Error::LettreError(err)
+    }
+}
+
+impl From<mime::FromStrError> for Error {
+    fn from(err: mime::FromStrError) -> Error {
+        Error::FromStr(err)
+    }
+}
+
+#[macro_export]
+macro_rules! bail {
+    ($e:expr) => {
+        return Err($crate::error::Error::Message($e.to_string()));
+    };
+    ($fmt:expr, $($arg:tt)+) => {
+        return Err($crate::error::Error::Message(format!($fmt, $($arg)+)));
+    };
+}
+
+#[macro_export]
+macro_rules! format_err {
+    ($e:expr) => {
+        $crate::error::Error::Message($e.to_string());
+    };
+    ($fmt:expr, $($arg:tt)+) => {
+        $crate::error::Error::Message(format!($fmt, $($arg)+));
+    };
+}
+
+#[macro_export(local_inner_macros)]
+macro_rules! ensure {
+    ($cond:expr, $e:expr) => {
+        if !($cond) {
+            bail!($e);
+        }
+    };
+    ($cond:expr, $fmt:expr, $($arg:tt)+) => {
+        if !($cond) {
+            bail!($fmt, $($arg)+);
+        }
+    };
+}
 
 #[macro_export]
 macro_rules! ensure_eq {

src/events.rs

@@ -4,7 +4,6 @@ use std::path::PathBuf;
 
 use strum::EnumProperty;
 
-use crate::chat::ChatId;
 use crate::message::MsgId;
 
 impl Event {
@@ -22,38 +21,56 @@ pub enum Event {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// @return 0
     #[strum(props(id = "100"))]
     Info(String),
 
     /// Emitted when SMTP connection is established and login was successful.
+    ///
+    /// @return 0
     #[strum(props(id = "101"))]
     SmtpConnected(String),
 
     /// Emitted when IMAP connection is established and login was successful.
+    ///
+    /// @return 0
     #[strum(props(id = "102"))]
     ImapConnected(String),
 
     /// Emitted when a message was successfully sent to the SMTP server.
+    ///
+    /// @return 0
     #[strum(props(id = "103"))]
     SmtpMessageSent(String),
 
     /// Emitted when an IMAP message has been marked as deleted
+    ///
+    /// @return 0
     #[strum(props(id = "104"))]
     ImapMessageDeleted(String),
 
     /// Emitted when an IMAP message has been moved
+    ///
+    /// @return 0
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
     /// Emitted when an IMAP folder was emptied
+    ///
+    /// @return 0
     #[strum(props(id = "106"))]
     ImapFolderEmptied(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -61,6 +78,8 @@ pub enum Event {
     /// Passed to the callback given to dc_context_new().
     ///
     /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// @return 0
     #[strum(props(id = "300"))]
     Warning(String),
 
@@ -75,6 +94,8 @@ pub enum Event {
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
+    ///
+    /// @return
     #[strum(props(id = "400"))]
     Error(String),
 
@@ -91,6 +112,8 @@ pub enum Event {
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
     /// instead of the string from data2.
+    ///
+    /// @return 0
     #[strum(props(id = "401"))]
     ErrorNetwork(String),
 
@@ -99,6 +122,8 @@ pub enum Event {
     /// dc_set_chat_name(), dc_set_chat_profile_image(),
     /// dc_add_contact_to_chat(), dc_remove_contact_from_chat(),
     /// dc_send_text_msg() or another sending function.
+    ///
+    /// @return 0
     #[strum(props(id = "410"))]
     ErrorSelfNotInGroup(String),
 
@@ -107,41 +132,54 @@ pub enum Event {
     /// - Messages sent, received or removed
     /// - Chats created, deleted or archived
     /// - A draft has been set
+    ///
+    /// @return 0
     #[strum(props(id = "2000"))]
-    MsgsChanged { chat_id: ChatId, msg_id: MsgId },
+    MsgsChanged { chat_id: u32, msg_id: MsgId },
 
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
     /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
+    ///
+    /// @return 0
     #[strum(props(id = "2005"))]
-    IncomingMsg { chat_id: ChatId, msg_id: MsgId },
+    IncomingMsg { chat_id: u32, msg_id: MsgId },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
+    ///
+    /// @return 0
     #[strum(props(id = "2010"))]
-    MsgDelivered { chat_id: ChatId, msg_id: MsgId },
+    MsgDelivered { chat_id: u32, msg_id: MsgId },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see dc_msg_get_state().
+    ///
+    /// @return 0
     #[strum(props(id = "2012"))]
-    MsgFailed { chat_id: ChatId, msg_id: MsgId },
+    MsgFailed { chat_id: u32, msg_id: MsgId },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
+    ///
+    /// @return 0
     #[strum(props(id = "2015"))]
-    MsgRead { chat_id: ChatId, msg_id: MsgId },
+    MsgRead { chat_id: u32, msg_id: MsgId },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// @return 0
     #[strum(props(id = "2020"))]
-    ChatModified(ChatId),
+    ChatModified(u32),
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
+    /// @return 0
     #[strum(props(id = "2030"))]
     ContactsChanged(Option<u32>),
 
@@ -150,12 +188,14 @@ pub enum Event {
     /// @param data1 (u32) contact_id of the contact for which the location has changed.
     ///     If the locations of several contacts have been changed,
     ///     eg. after calling dc_delete_all_locations(), this parameter is set to `None`.
+    /// @return 0
     #[strum(props(id = "2035"))]
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
+    /// @return 0
     #[strum(props(id = "2041"))]
     ConfigureProgress(usize),
 
@@ -163,6 +203,7 @@ pub enum Event {
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
+    /// @return 0
     #[strum(props(id = "2051"))]
     ImexProgress(usize),
 
@@ -173,6 +214,7 @@ pub enum Event {
     /// services.
     ///
     /// @param data2 0
+    /// @return 0
     #[strum(props(id = "2052"))]
     ImexFileWritten(PathBuf),
 
@@ -188,6 +230,7 @@ pub enum Event {
     ///     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
+    /// @return 0
     #[strum(props(id = "2060"))]
     SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
@@ -199,6 +242,14 @@ pub enum Event {
     /// @param data2 (int) Progress as:
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
+    /// @return 0
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
+
+    /// This event is sent out to the inviter when a joiner successfully joined a group.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    /// @return 0
+    #[strum(props(id = "2062"))]
+    SecurejoinMemberAdded { chat_id: u32, contact_id: u32 },
 }

src/headerdef.rs

@@ -1,95 +0,0 @@
-use crate::strum::AsStaticRef;
-use mailparse::{MailHeader, MailHeaderMap};
-
-#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
-#[strum(serialize_all = "kebab_case")]
-#[allow(dead_code)]
-pub enum HeaderDef {
-    MessageId,
-    Subject,
-    Date,
-    From_,
-    To,
-    Cc,
-    Disposition,
-    OriginalMessageId,
-
-    /// Delta Chat extension for message IDs in combined MDNs
-    AdditionalMessageIds,
-
-    ListId,
-    References,
-    InReplyTo,
-    Precedence,
-    ChatVersion,
-    ChatGroupId,
-    ChatGroupName,
-    ChatGroupNameChanged,
-    ChatVerified,
-    ChatGroupAvatar,
-    ChatUserAvatar,
-    ChatVoiceMessage,
-    ChatGroupMemberRemoved,
-    ChatGroupMemberAdded,
-    ChatContent,
-    ChatDuration,
-    ChatDispositionNotificationTo,
-    Autocrypt,
-    AutocryptSetupMessage,
-    SecureJoin,
-    SecureJoinGroup,
-    SecureJoinFingerprint,
-    SecureJoinInvitenumber,
-    SecureJoinAuth,
-    _TestHeader,
-}
-
-impl HeaderDef {
-    /// Returns the corresponding Event id.
-    pub fn get_headername(&self) -> &'static str {
-        self.as_static()
-    }
-}
-
-pub trait HeaderDefMap {
-    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
-}
-
-impl HeaderDefMap for [MailHeader<'_>] {
-    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
-        self.get_first_value(headerdef.get_headername())
-    }
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
-        self.get_first_header(headerdef.get_headername())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    /// Test that kebab_case serialization works as expected
-    fn kebab_test() {
-        assert_eq!(HeaderDef::From_.get_headername(), "from");
-
-        assert_eq!(HeaderDef::_TestHeader.get_headername(), "test-header");
-    }
-
-    #[test]
-    /// Test that headers are parsed case-insensitively
-    fn test_get_header_value_case() {
-        let (headers, _) =
-            mailparse::parse_headers(b"fRoM: Bob\naUtoCryPt-SeTup-MessAge: v99").unwrap();
-        assert_eq!(
-            headers.get_header_value(HeaderDef::AutocryptSetupMessage),
-            Some("v99".to_string())
-        );
-        assert_eq!(
-            headers.get_header_value(HeaderDef::From_),
-            Some("Bob".to_string())
-        );
-        assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None);
-    }
-}

src/imap/client.rs

@@ -1,104 +0,0 @@
-use async_imap::{
-    error::{Error as ImapError, Result as ImapResult},
-    Client as ImapClient,
-};
-use async_native_tls::TlsStream;
-use async_std::net::{self, TcpStream};
-
-use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
-
-#[derive(Debug)]
-pub(crate) enum Client {
-    Secure(ImapClient<TlsStream<TcpStream>>),
-    Insecure(ImapClient<TcpStream>),
-}
-
-impl Client {
-    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
-        addr: A,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
-        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
-        let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client::Secure(client))
-    }
-
-    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
-
-        let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client::Insecure(client))
-    }
-
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
-        match self {
-            Client::Insecure(client) => {
-                let tls = dc_build_tls(certificate_checks);
-                let client_sec = client.secure(domain, tls).await?;
-
-                Ok(Client::Secure(client_sec))
-            }
-            // Nothing to do
-            Client::Secure(_) => Ok(self),
-        }
-    }
-
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
-        self,
-        auth_type: S,
-        authenticator: &A,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
-        self,
-        username: U,
-        password: P,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-}

src/imap/idle.rs

@@ -1,56 +1,39 @@
 use super::Imap;
 
-use async_imap::extensions::idle::{Handle as ImapIdleHandle, IdleResponse};
-use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
+use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
 use async_std::task;
 use std::sync::atomic::Ordering;
 use std::time::{Duration, SystemTime};
 
 use crate::context::Context;
+use crate::imap_client::*;
 
 use super::select_folder;
-use super::session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
-    IdleProtocolFailed(#[from] async_imap::error::Error),
+    #[fail(display = "IMAP IDLE protocol failed to init/complete")]
+    IdleProtocolFailed(#[cause] async_imap::error::Error),
 
-    #[error("IMAP IDLE protocol timed out")]
-    IdleTimeout(#[from] async_std::future::TimeoutError),
-
-    #[error("IMAP server does not have IDLE capability")]
+    #[fail(display = "IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
+    #[fail(display = "IMAP select folder error")]
+    SelectFolderError(#[cause] select_folder::Error),
 
-    #[error("Setup handle error")]
-    SetupHandleError(#[from] super::Error),
+    #[fail(display = "IMAP error")]
+    ImapError(#[cause] async_imap::error::Error),
+
+    #[fail(display = "Setup handle error")]
+    SetupHandleError(#[cause] super::Error),
 }
 
-#[derive(Debug)]
-pub(crate) enum IdleHandle {
-    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
-    Insecure(ImapIdleHandle<TcpStream>),
-}
-
-impl Session {
-    pub fn idle(self) -> IdleHandle {
-        match self {
-            Session::Secure(i) => {
-                let h = i.idle();
-                IdleHandle::Secure(h)
-            }
-            Session::Insecure(i) => {
-                let h = i.idle();
-                IdleHandle::Insecure(h)
-            }
-        }
+impl From<select_folder::Error> for Error {
+    fn from(err: select_folder::Error) -> Error {
+        Error::SelectFolderError(err)
     }
 }
 
@@ -65,7 +48,9 @@ impl Imap {
                 return Err(Error::IdleAbilityMissing);
             }
 
-            self.setup_handle_if_needed(context).await?;
+            self.setup_handle_if_needed(context)
+                .await
+                .map_err(Error::SetupHandleError)?;
 
             self.select_folder(context, watch_folder.clone()).await?;
 
@@ -76,7 +61,9 @@ impl Imap {
                     // BEWARE: If you change the Secure branch you
                     // typically also need to change the Insecure branch.
                     IdleHandle::Secure(mut handle) => {
-                        handle.init().await?;
+                        if let Err(err) = handle.init().await {
+                            return Err(Error::IdleProtocolFailed(err));
+                        }
 
                         let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
                         *self.interrupt.lock().await = Some(interrupt);
@@ -90,34 +77,21 @@ impl Imap {
                         } else {
                             info!(context, "Idle entering wait-on-remote state");
                             match idle_wait.await {
-                                Ok(IdleResponse::NewData(_)) => {
+                                IdleResponse::NewData(_) => {
                                     info!(context, "Idle has NewData");
                                 }
                                 // TODO: idle_wait does not distinguish manual interrupts
                                 // from Timeouts if we would know it's a Timeout we could bail
                                 // directly and reconnect .
-                                Ok(IdleResponse::Timeout) => {
+                                IdleResponse::Timeout => {
                                     info!(context, "Idle-wait timeout or interruption");
                                 }
-                                Ok(IdleResponse::ManualInterrupt) => {
+                                IdleResponse::ManualInterrupt => {
                                     info!(context, "Idle wait was interrupted");
                                 }
-                                Err(err) => {
-                                    warn!(context, "Idle wait errored: {:?}", err);
-                                }
                             }
                         }
-                        // if we can't properly terminate the idle
-                        // protocol let's break the connection.
-                        let res =
-                            async_std::future::timeout(Duration::from_secs(15), handle.done())
-                                .await
-                                .map_err(|err| {
-                                    self.trigger_reconnect();
-                                    Error::IdleTimeout(err)
-                                })?;
-
-                        match res {
+                        match handle.done().await {
                             Ok(session) => {
                                 *self.session.lock().await = Some(Session::Secure(session));
                             }
@@ -131,7 +105,9 @@ impl Imap {
                         }
                     }
                     IdleHandle::Insecure(mut handle) => {
-                        handle.init().await?;
+                        if let Err(err) = handle.init().await {
+                            return Err(Error::IdleProtocolFailed(err));
+                        }
 
                         let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
                         *self.interrupt.lock().await = Some(interrupt);
@@ -145,34 +121,21 @@ impl Imap {
                         } else {
                             info!(context, "Idle entering wait-on-remote state");
                             match idle_wait.await {
-                                Ok(IdleResponse::NewData(_)) => {
+                                IdleResponse::NewData(_) => {
                                     info!(context, "Idle has NewData");
                                 }
                                 // TODO: idle_wait does not distinguish manual interrupts
                                 // from Timeouts if we would know it's a Timeout we could bail
                                 // directly and reconnect .
-                                Ok(IdleResponse::Timeout) => {
+                                IdleResponse::Timeout => {
                                     info!(context, "Idle-wait timeout or interruption");
                                 }
-                                Ok(IdleResponse::ManualInterrupt) => {
+                                IdleResponse::ManualInterrupt => {
                                     info!(context, "Idle wait was interrupted");
                                 }
-                                Err(err) => {
-                                    warn!(context, "Idle wait errored: {:?}", err);
-                                }
                             }
                         }
-                        // if we can't properly terminate the idle
-                        // protocol let's break the connection.
-                        let res =
-                            async_std::future::timeout(Duration::from_secs(15), handle.done())
-                                .await
-                                .map_err(|err| {
-                                    self.trigger_reconnect();
-                                    Error::IdleTimeout(err)
-                                })?;
-
-                        match res {
+                        match handle.done().await {
                             Ok(session) => {
                                 *self.session.lock().await = Some(Session::Insecure(session));
                             }
@@ -255,7 +218,7 @@ impl Imap {
                 "IMAP-fake-IDLE done after {:.4}s",
                 SystemTime::now()
                     .duration_since(fake_idle_start_time)
-                    .unwrap_or_default()
+                    .unwrap()
                     .as_millis() as f64
                     / 1000.,
             );

src/imap/select_folder.rs

@@ -4,53 +4,25 @@ use crate::context::Context;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("IMAP Could not obtain imap-session object.")]
+    #[fail(display = "IMAP Could not obtain imap-session object.")]
     NoSession,
 
-    #[error("IMAP Connection Lost or no connection established")]
+    #[fail(display = "IMAP Connection Lost or no connection established")]
     ConnectionLost,
 
-    #[error("IMAP Folder name invalid: {0}")]
+    #[fail(display = "IMAP Folder name invalid: {:?}", _0)]
     BadFolderName(String),
 
-    #[error("IMAP close/expunge failed")]
-    CloseExpungeFailed(#[from] async_imap::error::Error),
+    #[fail(display = "IMAP close/expunge failed: {}", _0)]
+    CloseExpungeFailed(#[cause] async_imap::error::Error),
 
-    #[error("IMAP other error: {0}")]
+    #[fail(display = "IMAP other error: {:?}", _0)]
     Other(String),
 }
 
 impl Imap {
-    /// Issues a CLOSE command to expunge selected folder.
-    ///
-    /// CLOSE is considerably faster than an EXPUNGE, see
-    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&self, context: &Context) -> Result<()> {
-        if let Some(ref folder) = self.config.read().await.selected_folder {
-            info!(context, "Expunge messages in \"{}\".", folder);
-
-            if let Some(ref mut session) = &mut *self.session.lock().await {
-                match session.close().await {
-                    Ok(_) => {
-                        info!(context, "close/expunge succeeded");
-                    }
-                    Err(err) => {
-                        self.trigger_reconnect();
-                        return Err(Error::CloseExpungeFailed(err));
-                    }
-                }
-            } else {
-                return Err(Error::NoSession);
-            }
-        }
-        let mut cfg = self.config.write().await;
-        cfg.selected_folder = None;
-        cfg.selected_folder_needs_expunge = false;
-        Ok(())
-    }
-
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -66,14 +38,39 @@ impl Imap {
             return Err(Error::NoSession);
         }
 
-        let needs_expunge = self.config.read().await.selected_folder_needs_expunge;
-        if needs_expunge {
-            self.close_folder(context).await?;
+        // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
+        // if there is _no_ new folder, we continue as we might want to expunge below.
+        if let Some(ref folder) = folder {
+            if let Some(ref selected_folder) = self.config.read().await.selected_folder {
+                if folder.as_ref() == selected_folder {
+                    return Ok(());
+                }
+            }
         }
 
-        let folder_str: Option<&str> = folder.as_ref().map(|x| x.as_ref());
-        if self.config.read().await.selected_folder.as_deref() == folder_str {
-            return Ok(());
+        // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
+        let needs_expunge = { self.config.read().await.selected_folder_needs_expunge };
+        if needs_expunge {
+            if let Some(ref folder) = self.config.read().await.selected_folder {
+                info!(context, "Expunge messages in \"{}\".", folder);
+
+                // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see
+                // https://tools.ietf.org/html/rfc3501#section-6.4.2
+                if let Some(ref mut session) = &mut *self.session.lock().await {
+                    match session.close().await {
+                        Ok(_) => {
+                            info!(context, "close/expunge succeeded");
+                        }
+                        Err(err) => {
+                            self.trigger_reconnect();
+                            return Err(Error::CloseExpungeFailed(err));
+                        }
+                    }
+                } else {
+                    return Err(Error::NoSession);
+                }
+            }
+            self.config.write().await.selected_folder_needs_expunge = false;
         }
 
         // select new folder

src/imap_client.rs

@@ -1,11 +1,21 @@
 use async_imap::{
-    error::Result as ImapResult,
+    error::{Error as ImapError, Result as ImapResult},
+    extensions::idle::Handle as ImapIdleHandle,
     types::{Capabilities, Fetch, Mailbox, Name},
-    Session as ImapSession,
+    Client as ImapClient, Session as ImapSession,
 };
-use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
+use async_std::net::{self, TcpStream};
 use async_std::prelude::*;
+use async_std::sync::Arc;
+use async_tls::client::TlsStream;
+
+use crate::login_param::{dc_build_tls_config, CertificateChecks};
+
+#[derive(Debug)]
+pub(crate) enum Client {
+    Secure(ImapClient<TlsStream<TcpStream>>),
+    Insecure(ImapClient<TcpStream>),
+}
 
 #[derive(Debug)]
 pub(crate) enum Session {
@@ -13,6 +23,104 @@ pub(crate) enum Session {
     Insecure(ImapSession<TcpStream>),
 }
 
+#[derive(Debug)]
+pub(crate) enum IdleHandle {
+    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
+    Insecure(ImapIdleHandle<TcpStream>),
+}
+
+impl Client {
+    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
+        addr: A,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+        let tls_config = dc_build_tls_config(certificate_checks);
+        let tls_connector: async_tls::TlsConnector = Arc::new(tls_config).into();
+        let tls_stream = tls_connector.connect(domain.as_ref(), stream)?.await?;
+        let mut client = ImapClient::new(tls_stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+
+        let _greeting = client
+            .read_response()
+            .await
+            .expect("failed to read greeting");
+
+        Ok(Client::Secure(client))
+    }
+
+    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+
+        let mut client = ImapClient::new(stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        let _greeting = client
+            .read_response()
+            .await
+            .expect("failed to read greeting");
+
+        Ok(Client::Insecure(client))
+    }
+
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
+        match self {
+            Client::Insecure(client) => {
+                let tls_config = dc_build_tls_config(certificate_checks);
+                let tls: async_tls::TlsConnector = Arc::new(tls_config).into();
+
+                let client_sec = client.secure(domain, &tls).await?;
+
+                Ok(Client::Secure(client_sec))
+            }
+            // Nothing to do
+            Client::Secure(_) => Ok(self),
+        }
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+}
+
 impl Session {
     pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
         let res = match self {
@@ -123,6 +231,19 @@ impl Session {
         Ok(res)
     }
 
+    pub fn idle(self) -> IdleHandle {
+        match self {
+            Session::Secure(i) => {
+                let h = i.idle();
+                IdleHandle::Secure(h)
+            }
+            Session::Insecure(i) => {
+                let h = i.idle();
+                IdleHandle::Insecure(h)
+            }
+        }
+    }
+
     pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
     where
         S1: AsRef<str>,

src/imex.rs

@@ -8,8 +8,8 @@ use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
 use crate::chat;
-use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
+use crate::configure::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
@@ -17,7 +17,7 @@ use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
 use crate::job::*;
-use crate::key::{self, DcKey, Key, SignedSecretKey};
+use crate::key::*;
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -33,19 +33,16 @@ pub enum ImexMode {
     /// and `private-key-default.asc`, if there are more keys, they are written to files as
     /// `public-key-<id>.asc` and `private-key-<id>.asc`
     ExportSelfKeys = 1,
-
     /// Import private keys found in the directory given as `param1`.
     /// The last imported key is made the default keys unless its name contains the string `legacy`.
     /// Public keys are not imported.
     ImportSelfKeys = 2,
-
     /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
     /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
     /// the format is `delta-chat.<day>-<number>.bak`
     ExportBackup = 11,
-
     /// `param1` is the file (not: directory) to import. The file is normally
     /// created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
     /// is only possible as long as the context is not configured or used in another way.
@@ -136,16 +133,14 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 
     let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF)?;
     msg = Message::default();
-    msg.viewtype = Viewtype::File;
+    msg.type_0 = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
 
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param.set_int(
-        Param::ForcePlaintext,
-        ForcePlaintext::NoAutocryptHeader as i32,
-    );
+    msg.param
+        .set_int(Param::ForcePlaintext, DC_FP_NO_AUTOCRYPT_HEADER);
 
     ensure!(!context.shall_stop_ongoing(), "canceled");
     let msg_id = chat::send_msg(context, chat_id, &mut msg)?;
@@ -175,7 +170,9 @@ pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String>
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let private_key = Key::from(SignedSecretKey::load_self(context)?);
+    let self_addr = e2ee::ensure_secret_key_exists(context)?;
+    let private_key = Key::from_self_private(context, self_addr, &context.sql)
+        .ok_or_else(|| format_err!("Failed to get private key."))?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled) {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
@@ -286,6 +283,7 @@ fn set_self_key(
         .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
 
     ensure!(keys.is_some(), "Not a valid private key");
+
     let (private_key, public_key, header) = keys.unwrap();
     let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
     match preferencrypt.map(|s| s.as_str()) {
@@ -310,26 +308,34 @@ fn set_self_key(
 
     let self_addr = context.get_config(Config::ConfiguredAddr);
     ensure!(self_addr.is_some(), "Missing self addr");
-    let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
 
-    let (public, secret) = match (public_key, private_key) {
-        (Key::Public(p), Key::Secret(s)) => (p, s),
-        _ => bail!("wrong keys unpacked"),
-    };
-    let keypair = pgp::KeyPair {
-        addr,
-        public,
-        secret,
-    };
-    key::store_self_keypair(
+    // XXX maybe better make dc_key_save_self_keypair delete things
+    sql::execute(
         context,
-        &keypair,
-        if set_default {
-            key::KeyPairUse::Default
-        } else {
-            key::KeyPairUse::ReadOnly
-        },
+        &context.sql,
+        "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
+        params![public_key.to_bytes(), private_key.to_bytes()],
     )?;
+
+    if set_default {
+        sql::execute(
+            context,
+            &context.sql,
+            "UPDATE keypairs SET is_default=0;",
+            params![],
+        )?;
+    }
+
+    if !dc_key_save_self_keypair(
+        context,
+        &public_key,
+        &private_key,
+        self_addr.unwrap_or_default(),
+        set_default,
+        &context.sql,
+    ) {
+        bail!("Cannot save keypair, internal key-state possibly corrupted now!");
+    }
     Ok(())
 }
 
@@ -374,7 +380,7 @@ pub fn JobImexImap(context: &Context, job: &Job) -> Result<()> {
             context.free_ongoing();
             bail!("Cannot create private key or private key not available.");
         } else {
-            dc_create_folder(context, &param)?;
+            dc_create_folder(context, &param);
         }
     }
     let path = Path::new(param);
@@ -411,7 +417,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     );
 
     ensure!(
-        !context.is_configured(),
+        !dc_is_configured(context),
         "Cannot import backups to accounts in use."
     );
     context.sql.close(&context);
@@ -432,8 +438,6 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
         "could not re-open db"
     );
 
-    delete_and_reset_all_device_msgs(&context)?;
-
     let total_files_cnt = context
         .sql
         .query_get_value::<_, isize>(context, "SELECT COUNT(*) FROM backup_blobs;", params![])
@@ -734,6 +738,7 @@ fn export_key_to_asc_file(
 #[cfg(test)]
 mod tests {
     use super::*;
+
     use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
     use crate::test_utils::*;
     use ::pgp::armor::BlockType;
@@ -754,6 +759,7 @@ mod tests {
         assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n"));
         assert!(msg.contains("Passphrase-Format: numeric9x4\r\n"));
         assert!(msg.contains("Passphrase-Begin: he\n"));
+        assert!(msg.contains("==\n"));
         assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 
@@ -787,7 +793,8 @@ mod tests {
     #[test]
     fn test_export_key_to_asc_file() {
         let context = dummy_context();
-        let key = Key::from(alice_keypair().public);
+        let base64 = include_str!("../test-data/key/public.asc");
+        let key = Key::from_base64(base64, KeyType::Public).unwrap();
         let blobdir = "$BLOBDIR";
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key).is_ok());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();

src/job_thread.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, Condvar, Mutex};
 
 use crate::context::Context;
-use crate::error::{format_err, Result};
+use crate::error::{Error, Result};
 use crate::imap::Imap;
 
 #[derive(Debug)]
@@ -99,21 +99,25 @@ impl JobThread {
 
     async fn connect_and_fetch(&mut self, context: &Context) -> Result<()> {
         let prefix = format!("{}-fetch", self.name);
-        self.imap.connect_configured(context)?;
-        if let Some(watch_folder) = self.get_watch_folder(context) {
-            let start = std::time::Instant::now();
-            info!(context, "{} started...", prefix);
-            let res = self
-                .imap
-                .fetch(context, &watch_folder)
-                .await
-                .map_err(Into::into);
-            let elapsed = start.elapsed().as_millis();
-            info!(context, "{} done in {:.3} ms.", prefix, elapsed);
+        match self.imap.connect_configured(context) {
+            Ok(()) => {
+                if let Some(watch_folder) = self.get_watch_folder(context) {
+                    let start = std::time::Instant::now();
+                    info!(context, "{} started...", prefix);
+                    let res = self
+                        .imap
+                        .fetch(context, &watch_folder)
+                        .await
+                        .map_err(Into::into);
+                    let elapsed = start.elapsed().as_millis();
+                    info!(context, "{} done in {:.3} ms.", prefix, elapsed);
 
-            res
-        } else {
-            Err(format_err!("WatchFolder not found: not-set"))
+                    res
+                } else {
+                    Err(Error::WatchFolderNotFound("not-set".to_string()))
+                }
+            }
+            Err(err) => Err(crate::error::Error::Message(err.to_string())),
         }
     }
 
@@ -139,7 +143,7 @@ impl JobThread {
             if state.jobs_needed {
                 info!(
                     context,
-                    "{}-IDLE will not be started as it was interrupted while not idling.",
+                    "{}-IDLE will not be started as it was interrupted while not ideling.",
                     self.name,
                 );
                 state.jobs_needed = false;

src/key.rs

@@ -4,169 +4,14 @@ use std::collections::BTreeMap;
 use std::io::Cursor;
 use std::path::Path;
 
-use num_traits::FromPrimitive;
-use pgp::composed::Deserializable;
+use pgp::composed::{Deserializable, SignedPublicKey, SignedSecretKey};
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
 
-use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
-use crate::sql;
-
-// Re-export key types
-pub use crate::pgp::KeyPair;
-pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
-
-/// Error type for deltachat key handling.
-#[derive(Debug, thiserror::Error)]
-#[non_exhaustive]
-pub enum Error {
-    #[error("Could not decode base64")]
-    Base64Decode(#[from] base64::DecodeError),
-    #[error("rPGP error: {}", _0)]
-    Pgp(#[from] pgp::errors::Error),
-    #[error("Failed to generate PGP key: {}", _0)]
-    Keygen(#[from] crate::pgp::PgpKeygenError),
-    #[error("Failed to load key: {}", _0)]
-    LoadKey(#[from] sql::Error),
-    #[error("Failed to save generated key: {}", _0)]
-    StoreKey(#[from] SaveKeyError),
-    #[error("No address configured")]
-    NoConfiguredAddr,
-    #[error("Configured address is invalid: {}", _0)]
-    InvalidConfiguredAddr(#[from] InvalidEmailError),
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-/// Convenience trait for working with keys.
-///
-/// This trait is implemented for rPGP's [SignedPublicKey] and
-/// [SignedSecretKey] types and makes working with them a little
-/// easier in the deltachat world.
-pub trait DcKey: Serialize + Deserializable {
-    type KeyType: Serialize + Deserializable;
-
-    /// Create a key from some bytes.
-    fn from_slice(bytes: &[u8]) -> Result<Self::KeyType> {
-        Ok(<Self::KeyType as Deserializable>::from_bytes(Cursor::new(
-            bytes,
-        ))?)
-    }
-
-    /// Create a key from a base64 string.
-    fn from_base64(data: &str) -> Result<Self::KeyType> {
-        // strip newlines and other whitespace
-        let cleaned: String = data.trim().split_whitespace().collect();
-        let bytes = base64::decode(cleaned.as_bytes())?;
-        Self::from_slice(&bytes)
-    }
-
-    /// Load the users' default key from the database.
-    fn load_self(context: &Context) -> Result<Self::KeyType>;
-
-    /// Serialise the key to a base64 string.
-    fn to_base64(&self) -> String {
-        // Not using Serialize::to_bytes() to make clear *why* it is
-        // safe to ignore this error.
-        // Because we write to a Vec<u8> the io::Write impls never
-        // fail and we can hide this error.
-        let mut buf = Vec::new();
-        self.to_writer(&mut buf).unwrap();
-        base64::encode(&buf)
-    }
-}
-
-impl DcKey for SignedPublicKey {
-    type KeyType = SignedPublicKey;
-
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
-            SELECT public_key
-              FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1;
-            "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
-            Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
-                Ok(keypair.public)
-            }
-            Err(err) => Err(err.into()),
-        }
-    }
-}
-
-impl DcKey for SignedSecretKey {
-    type KeyType = SignedSecretKey;
-
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
-            SELECT private_key
-              FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1;
-            "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
-            Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
-                Ok(keypair.secret)
-            }
-            Err(err) => Err(err.into()),
-        }
-    }
-}
-
-fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context
-        .get_config(Config::ConfiguredAddr)
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
-    let addr = EmailAddress::new(&addr)?;
-    let _guard = context.generating_key_mutex.lock().unwrap();
-
-    // Check if the key appeared while we were waiting on the lock.
-    match context.sql.query_row(
-        r#"
-        SELECT public_key, private_key
-          FROM keypairs
-         WHERE addr=?1
-           AND is_default=1;
-        "#,
-        params![addr],
-        |row| Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?)),
-    ) {
-        Ok((pub_bytes, sec_bytes)) => Ok(KeyPair {
-            addr,
-            public: SignedPublicKey::from_slice(&pub_bytes)?,
-            secret: SignedSecretKey::from_slice(&sec_bytes)?,
-        }),
-        Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
-            let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType))
-                .unwrap_or_default();
-            info!(context, "Generating keypair with type {}", keytype);
-            let keypair = crate::pgp::create_keypair(addr, keytype)?;
-            store_self_keypair(context, &keypair, KeyPairUse::Default)?;
-            info!(
-                context,
-                "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
-            );
-            Ok(keypair)
-        }
-        Err(err) => Err(err.into()),
-    }
-}
+use crate::dc_tools::*;
+use crate::sql::{self, Sql};
 
 /// Cryptographic key
 #[derive(Debug, PartialEq, Eq, Clone)]
@@ -190,7 +35,7 @@ impl From<SignedSecretKey> for Key {
 impl std::convert::TryFrom<Key> for SignedSecretKey {
     type Error = ();
 
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
+    fn try_from(value: Key) -> Result<Self, Self::Error> {
         match value {
             Key::Public(_) => Err(()),
             Key::Secret(key) => Ok(key),
@@ -201,7 +46,7 @@ impl std::convert::TryFrom<Key> for SignedSecretKey {
 impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
     type Error = ();
 
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
+    fn try_from(value: &'a Key) -> Result<Self, Self::Error> {
         match value {
             Key::Public(_) => Err(()),
             Key::Secret(key) => Ok(key),
@@ -212,7 +57,7 @@ impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
 impl std::convert::TryFrom<Key> for SignedPublicKey {
     type Error = ();
 
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
+    fn try_from(value: Key) -> Result<Self, Self::Error> {
         match value {
             Key::Public(key) => Ok(key),
             Key::Secret(_) => Err(()),
@@ -223,7 +68,7 @@ impl std::convert::TryFrom<Key> for SignedPublicKey {
 impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
     type Error = ();
 
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
+    fn try_from(value: &'a Key) -> Result<Self, Self::Error> {
         match value {
             Key::Public(key) => Ok(key),
             Key::Secret(_) => Err(()),
@@ -247,7 +92,7 @@ impl Key {
         if bytes.is_empty() {
             return None;
         }
-        let res: std::result::Result<Key, _> = match key_type {
+        let res: Result<Key, _> = match key_type {
             KeyType::Public => SignedPublicKey::from_bytes(Cursor::new(bytes)).map(Into::into),
             KeyType::Private => SignedSecretKey::from_bytes(Cursor::new(bytes)).map(Into::into),
         };
@@ -266,7 +111,7 @@ impl Key {
         key_type: KeyType,
     ) -> Option<(Self, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        let res: std::result::Result<(Key, _), _> = match key_type {
+        let res: Result<(Key, _), _> = match key_type {
             KeyType::Public => SignedPublicKey::from_armor_single(Cursor::new(bytes))
                 .map(|(k, h)| (Into::into(k), h)),
             KeyType::Private => SignedSecretKey::from_armor_single(Cursor::new(bytes))
@@ -282,6 +127,43 @@ impl Key {
         }
     }
 
+    pub fn from_base64(encoded_data: &str, key_type: KeyType) -> Option<Self> {
+        // strip newlines and other whitespace
+        let cleaned: String = encoded_data.trim().split_whitespace().collect();
+        let bytes = cleaned.as_bytes();
+        base64::decode(bytes)
+            .ok()
+            .and_then(|decoded| Self::from_slice(&decoded, key_type))
+    }
+
+    pub fn from_self_public(
+        context: &Context,
+        self_addr: impl AsRef<str>,
+        sql: &Sql,
+    ) -> Option<Self> {
+        let addr = self_addr.as_ref();
+
+        sql.query_get_value(
+            context,
+            "SELECT public_key FROM keypairs WHERE addr=? AND is_default=1;",
+            &[addr],
+        )
+        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Public))
+    }
+
+    pub fn from_self_private(
+        context: &Context,
+        self_addr: impl AsRef<str>,
+        sql: &Sql,
+    ) -> Option<Self> {
+        sql.query_get_value(
+            context,
+            "SELECT private_key FROM keypairs WHERE addr=? AND is_default=1;",
+            &[self_addr.as_ref()],
+        )
+        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Private))
+    }
+
     pub fn to_bytes(&self) -> Vec<u8> {
         match self {
             Key::Public(k) => k.to_bytes().unwrap_or_default(),
@@ -360,95 +242,20 @@ impl Key {
     }
 }
 
-/// Use of a [KeyPair] for encryption or decryption.
-///
-/// This is used by [store_self_keypair] to know what kind of key is
-/// being saved.
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub enum KeyPairUse {
-    /// The default key used to encrypt new messages.
-    Default,
-    /// Only used to decrypt existing message.
-    ReadOnly,
-}
-
-/// Error saving a keypair to the database.
-#[derive(Debug, thiserror::Error)]
-#[error("SaveKeyError: {message}")]
-pub struct SaveKeyError {
-    message: String,
-    #[source]
-    cause: anyhow::Error,
-}
-
-impl SaveKeyError {
-    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
-        Self {
-            message: message.into(),
-            cause: cause.into(),
-        }
-    }
-}
-
-/// Store the keypair as an owned keypair for addr in the database.
-///
-/// This will save the keypair as keys for the given address.  The
-/// "self" here refers to the fact that this DC instance owns the
-/// keypair.  Usually `addr` will be [Config::ConfiguredAddr].
-///
-/// If either the public or private keys are already present in the
-/// database, this entry will be removed first regardless of the
-/// address associated with it.  Practically this means saving the
-/// same key again overwrites it.
-///
-/// [Config::ConfiguredAddr]: crate::config::Config::ConfiguredAddr
-pub fn store_self_keypair(
+pub fn dc_key_save_self_keypair(
     context: &Context,
-    keypair: &KeyPair,
-    default: KeyPairUse,
-) -> std::result::Result<(), SaveKeyError> {
-    // Everything should really be one transaction, more refactoring
-    // is needed for that.
-    let public_key = keypair
-        .public
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise public key", err))?;
-    let secret_key = keypair
-        .secret
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise secret key", err))?;
-    context
-        .sql
-        .execute(
-            "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
-            params![public_key, secret_key],
-        )
-        .map_err(|err| SaveKeyError::new("failed to remove old use of key", err))?;
-    if default == KeyPairUse::Default {
-        context
-            .sql
-            .execute("UPDATE keypairs SET is_default=0;", params![])
-            .map_err(|err| SaveKeyError::new("failed to clear default", err))?;
-    }
-    let is_default = match default {
-        KeyPairUse::Default => true,
-        KeyPairUse::ReadOnly => false,
-    };
-    context
-        .sql
-        .execute(
-            "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
-                VALUES (?,?,?,?,?);",
-            params![
-                keypair.addr.to_string(),
-                is_default as i32,
-                public_key,
-                secret_key,
-                time()
-            ],
-        )
-        .map(|_| ())
-        .map_err(|err| SaveKeyError::new("failed to insert keypair", err))
+    public_key: &Key,
+    private_key: &Key,
+    addr: impl AsRef<str>,
+    is_default: bool,
+    sql: &Sql,
+) -> bool {
+    sql::execute(
+        context,
+        sql,
+        "INSERT INTO keypairs (addr, is_default, public_key, private_key, created) VALUES (?,?,?,?,?);",
+        params![addr.as_ref(), is_default as i32, public_key.to_bytes(), private_key.to_bytes(), time()],
+    ).is_ok()
 }
 
 /// Make a fingerprint human-readable, in hex format.
@@ -480,14 +287,6 @@ pub fn dc_normalize_fingerprint(fp: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
-    use std::convert::TryFrom;
-
-    use lazy_static::lazy_static;
-
-    lazy_static! {
-        static ref KEYPAIR: KeyPair = alice_keypair();
-    }
 
     #[test]
     fn test_normalize_fingerprint() {
@@ -574,9 +373,9 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[test]
+    #[ignore] // is too expensive
     fn test_from_slice_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+        let (public_key, private_key) = crate::pgp::create_keypair("hello").unwrap();
 
         let binary = public_key.to_bytes();
         let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
@@ -609,62 +408,9 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[test]
-    fn test_load_self_existing() {
-        let alice = alice_keypair();
-        let t = dummy_context();
-        configure_alice_keypair(&t.ctx);
-        let pubkey = SignedPublicKey::load_self(&t.ctx).unwrap();
-        assert_eq!(alice.public, pubkey);
-        let seckey = SignedSecretKey::load_self(&t.ctx).unwrap();
-        assert_eq!(alice.secret, seckey);
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_public() {
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let key = SignedPublicKey::load_self(&t.ctx);
-        assert!(key.is_ok());
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_secret() {
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let key = SignedSecretKey::load_self(&t.ctx);
-        assert!(key.is_ok());
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_concurrent() {
-        use std::sync::Arc;
-        use std::thread;
-
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let ctx = Arc::new(t.ctx);
-        let ctx0 = Arc::clone(&ctx);
-        let thr0 = thread::spawn(move || SignedPublicKey::load_self(&ctx0));
-        let ctx1 = Arc::clone(&ctx);
-        let thr1 = thread::spawn(move || SignedPublicKey::load_self(&ctx1));
-        let res0 = thr0.join().unwrap();
-        let res1 = thr1.join().unwrap();
-        assert_eq!(res0.unwrap(), res1.unwrap());
-    }
-
-    #[test]
+    #[ignore] // is too expensive
     fn test_ascii_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+        let (public_key, private_key) = crate::pgp::create_keypair("hello").unwrap();
 
         let s = public_key.to_armored_string(None).unwrap();
         let (public_key2, _) =
@@ -677,51 +423,4 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
             Key::from_armored_string(&s, KeyType::Private).expect("invalid private key");
         assert_eq!(private_key, private_key2);
     }
-
-    #[test]
-    fn test_split_key() {
-        let private_key = Key::from(KEYPAIR.secret.clone());
-        let public_wrapped = private_key.split_key().unwrap();
-        let public = SignedPublicKey::try_from(public_wrapped).unwrap();
-        assert_eq!(public.primary_key, KEYPAIR.public.primary_key);
-    }
-
-    #[test]
-    fn test_save_self_key_twice() {
-        // Saving the same key twice should result in only one row in
-        // the keypairs table.
-        let t = dummy_context();
-        let nrows = || {
-            t.ctx
-                .sql
-                .query_get_value::<_, u32>(&t.ctx, "SELECT COUNT(*) FROM keypairs;", params![])
-                .unwrap()
-        };
-        assert_eq!(nrows(), 0);
-        store_self_keypair(&t.ctx, &KEYPAIR, KeyPairUse::Default).unwrap();
-        assert_eq!(nrows(), 1);
-        store_self_keypair(&t.ctx, &KEYPAIR, KeyPairUse::Default).unwrap();
-        assert_eq!(nrows(), 1);
-    }
-
-    // Convenient way to create a new key if you need one, run with
-    // `cargo test key::tests::gen_key`.
-    // #[test]
-    // fn gen_key() {
-    //     let name = "fiona";
-    //     let keypair = crate::pgp::create_keypair(
-    //         EmailAddress::new(&format!("{}@example.net", name)).unwrap(),
-    //     )
-    //     .unwrap();
-    //     std::fs::write(
-    //         format!("test-data/key/{}-public.asc", name),
-    //         keypair.public.to_base64(),
-    //     )
-    //     .unwrap();
-    //     std::fs::write(
-    //         format!("test-data/key/{}-secret.asc", name),
-    //         keypair.secret.to_base64(),
-    //     )
-    //     .unwrap();
-    // }
 }

src/keyring.rs

@@ -1,8 +1,8 @@
 use std::borrow::Cow;
 
-use crate::constants::KeyType;
+use crate::constants::*;
 use crate::context::Context;
-use crate::key::Key;
+use crate::key::*;
 use crate::sql::Sql;
 
 #[derive(Default, Clone, Debug)]

src/lib.rs

@@ -1,7 +1,21 @@
-#![forbid(unsafe_code)]
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![allow(clippy::match_bool)]
+// for now we hide warnings to not clutter/hide errors during "cargo clippy"
+#![allow(
+    clippy::type_complexity,
+    clippy::cognitive_complexity,
+    clippy::too_many_arguments,
+    clippy::block_in_if_condition_stmt,
+    clippy::large_enum_variant
+)]
+#![allow(
+    clippy::unreadable_literal,
+    clippy::needless_range_loop,
+    clippy::match_bool
+)]
+#![feature(ptr_wrapping_offset_from)]
 
+#[macro_use]
+extern crate failure_derive;
 #[macro_use]
 extern crate num_derive;
 #[macro_use]
@@ -15,32 +29,30 @@ extern crate strum_macros;
 extern crate debug_stub_derive;
 
 #[macro_use]
-pub mod log;
+mod log;
 #[macro_use]
 pub mod error;
 
-pub mod headerdef;
-
 pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
-mod blob;
+pub mod blob;
 pub mod chat;
 pub mod chatlist;
 pub mod config;
-mod configure;
+pub mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
 mod imap;
+mod imap_client;
 pub mod imex;
-#[macro_use]
 pub mod job;
 mod job_thread;
 pub mod key;
-mod keyring;
+pub mod keyring;
 pub mod location;
 mod login_param;
 pub mod lot;
@@ -51,18 +63,19 @@ pub mod oauth2;
 mod param;
 pub mod peerstate;
 pub mod pgp;
-pub mod provider;
 pub mod qr;
 pub mod securejoin;
-mod simplify;
 mod smtp;
 pub mod sql;
 pub mod stock;
 mod token;
 #[macro_use]
+mod wrapmime;
 mod dehtml;
 
+pub mod dc_array;
 pub mod dc_receive_imf;
+mod dc_simplify;
 pub mod dc_tools;
 
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed