more type fixes

add more tests for BlobObject::sanitise_name()

.circleci/config.yml

@@ -7,6 +7,10 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
+            
 
 restore-workspace: &restore-workspace
   attach_workspace:
@@ -134,12 +138,6 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests_rust:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_rust.sh
-    
   remote_tests_python:
     machine: true
     steps:
@@ -174,11 +172,6 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
-
       - remote_tests_python:
           filters:
             tags:
@@ -187,8 +180,9 @@ workflows:
       - remote_python_packaging:
           requires: 
              - remote_tests_python
-             - remote_tests_rust 
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 
@@ -197,6 +191,8 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -208,6 +204,8 @@ workflows:
 
       - build_doxygen:
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: Rust CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.43.1
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.43.1
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly, 1.43.1]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      with:
+        command:  check
+        args: --workspace --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --workspace
+          

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-12
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-12
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-12
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

ASYNC-API-TODO.txt

@@ -0,0 +1,54 @@
+
+Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
+
+- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
+
+APIs:
+    dc_context_new                 # opens the database
+    dc_open                             # FFI only
+      -> drop it and move parameters to dc_context_new()
+
+    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
+    dc_imex NEVER goes through the job system
+    dc_imex import_backup needs to ensure dc_stop_jobs()
+
+    dc_start_io                # start smtp/imap and job handling subsystems
+    dc_stop_io                 # stop smtp/imap and job handling subsystems
+    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
+
+    dc_close                            # FFI only
+      -> can be dropped
+    dc_context_unref
+
+    for ios share-extension:
+    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
+        0: message was sent
+        1: message failed to go out, is queued as a job to be retried later
+        2: message permanently failed?
+
+EVENT handling:
+    start a callback thread and call get_next_event() which is BLOCKING
+    it's fine to start this callback thread later, it will see all events.
+    Note that the core infinitely fills the internal queue if you never drain it.
+
+    FFI-get_next_event() returns NULL if the context is unrefed already?
+
+   sidenote: how python's callback thread does it currently:
+         CB-thread runs this while loop:
+         while not QUITFLAG:
+             ev = context.get_next_event( )
+             ...
+         So in order to shutdown properly one has to set QUITFLAG
+         before calling dc_stop_jobs() and dc_context_unref
+
+    event API:
+        get_data1_int
+        get_data2_int
+        get_data3_str
+
+
+- userdata likely only used for the callbacks, likely can be dropped, needs verification
+
+
+- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
+

CHANGELOG.md

@@ -1,5 +1,167 @@
 # Changelog 
 
+## 1.39.0
+
+- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
+
+- fix potential panics, update dependencies #1650 #1655
+
+
+## 1.38.0
+
+- fix sorting, esp. for multi-device
+
+
+## 1.37.0
+
+- improve ndn heuristics #1630
+
+- get oauth2 authorizer from provider-db #1641
+
+- removed linebreaks and spaces from generated qr-code #1631
+
+- more fixes #1633 #1635 #1636 #1637
+
+
+## 1.36.0
+
+- parse ndn (network delivery notification) reports
+  and report failed messages as such #1552 #1622 #1630
+
+- add oauth2 support for gsuite domains #1626
+
+- read image orientation from exif before recoding #1619
+
+- improve logging #1593 #1598
+
+- improve python and bot bindings #1583 #1609
+
+- improve imap logout #1595
+
+- fix sorting #1600 #1604
+
+- fix qr code generation #1631
+
+- update rustcrypto releases #1603
+
+- refactorings #1617
+
+
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
+## 1.34.0
+
+- new api for io, thread and event handling #1356,
+  see the example atop of `deltachat.h` to get an overview
+
+- LOTS of speed improvements due to async processing #1356
+
+- enable WAL mode for sqlite #1492
+
+- process incoming messages in bulk #1527
+
+- improve finding out the sent-folder #1488
+
+- several bug fixes
+
+
+## 1.33.0
+
+- let `dc_set_muted()` also mute one-to-one chats #1470
+
+- fix a bug that led to load and traffic if the server does not use sent-folder
+  #1472
+
+
+## 1.32.0
+
+- fix endless loop when trying to download messages with bad RFC Message-ID,
+  also be more reliable on similar errors #1463 #1466 #1462
+
+- fix bug with comma in contact request #1438
+
+- do not refer to hidden messages on replies #1459
+
+- improve error handling #1468 #1465 #1464
+
+
+## 1.31.0
+
+- always describe the context of the displayed error #1451
+
+- do not emit `DC_EVENT_ERROR` when message sending fails;
+  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
+
+- new config-option `media_quality` #1449
+
+- try over if writing message to database fails #1447
+
+
+## 1.30.0
+
+- expunge deleted messages #1440
+
+- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
+
+
+## 1.29.0
+
+- new config options `delete_device_after` and `delete_server_after`,
+  each taking an amount of seconds after which messages
+  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
+
+- new api `dc_estimate_deletion_cnt()` to estimate the effect
+  of `delete_device_after` and `delete_server_after`
+
+- use Ed25519 keys by default, these keys are much shorter
+  than RSA keys, which results in saving traffic and speed improvements #1362
+
+- improve message ellipsizing #1397 #1430
+
+- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
+
+- do not show badly formatted non-delta-messages as empty #1384
+
+- try over SMTP on potentially recoverable error 5.5.0 #1379
+
+- remove device-chat from forward-to-chat-list #1367
+
+- improve group-handling #1368
+
+- `dc_get_info()` returns uptime (how long the context is in use)
+
+- python improvements and adaptions #1408 #1415
+
+- log to the stdout and stderr in tests #1416
+
+- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
+
+- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
+
+- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
+
+
 ## 1.28.0
 
 - new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.28.0"
+version = "1.39.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,60 +12,67 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-reqwest = { version = "0.10.0", features = ["blocking", "json"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = "0.2"
+async-smtp = { version = "0.3" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.2"
-async-native-tls = "0.3.1"
-async-std = { version = "1.4", features = ["unstable"] }
-base64 = "0.11"
+async-imap = "0.3.1"
+async-native-tls = { version = "0.3.3" }
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
+kamadak-exif = "0.5"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.21", features = ["bundled"] }
-r2d2_sqlite = "0.13.0"
-r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
-thread-local-object = "0.1.0"
+strum = "0.18.0"
+strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.17.1"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.0"
+mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
-pretty_env_logger = "0.3.1"
-
-rustyline = { version = "4.1.0", optional = true }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
+async-trait = "0.1.31"
+url = "2.1.1"
+async-std-resolver = "0.19.5"
+sqlx = { git = "https://github.com/launchbadge/sqlx", branch = "master", features = ["runtime-async-std", "sqlite", "macros"] }
+libsqlite3-sys = { version = "0.18", features = ["bundled", "min_sqlite_version_3_7_16"] }
+
+pretty_env_logger = { version = "0.4.0", optional = true }
+log = { version = "0.4.8", optional = true }
+rustyline = { version = "4.1.0", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
+async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+smol = "0.1.11"
+log = "0.4.8"
 
 [workspace]
 members = [
@@ -76,15 +83,17 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
+required-features = ["repl"]
 
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
-required-features = ["rustyline"]
+required-features = ["repl"]
 
 
 [features]
-default = ["nightly"]
-vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
+default = []
+internals = []
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
-

README.md

@@ -9,7 +9,7 @@
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
 
 ```
-curl https://sh.rustup.rs -sSf | sh
+$ curl https://sh.rustup.rs -sSf | sh
 ```
 
 ## Using the CLI client
@@ -17,8 +17,9 @@ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-cargo run --example repl -- /path/to/db
+$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
+where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 
@@ -122,6 +123,7 @@ Language bindings are available for:
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
 - [Go](https://github.com/hugot/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library
@@ -131,4 +133,5 @@ or its language bindings:
 - [iOS](https://github.com/deltachat/deltachat-ios)
 - [Desktop](https://github.com/deltachat/deltachat-desktop)
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - several **Bots**

appveyor.yml

@@ -1,19 +0,0 @@
-environment:
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-12
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - rustc -vV
-  - cargo -vV
-
-build: false
-
-test_script:
-  - cargo test --release --all
-
-cache:
-  - target
-  - C:\Users\appveyor\.cargo\registry

ci_scripts/ci_upload.sh

@@ -37,7 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip
+pip3 install -U pip setuptools
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN

ci_scripts/cleanup_devpi_indices.py

@@ -48,7 +48,7 @@ def run():
         projectnames = get_projectnames(baseurl, username, indexname)
         if indexname == "master" or not indexname:
             continue
-        assert projectnames == ["deltachat"]
+        clear_index = not projectnames
         for projectname in projectnames:
             dates = get_release_dates(baseurl, username, indexname, projectname)
             if not dates:
@@ -60,8 +60,11 @@ def run():
                 date = datetime.datetime(*max(dates))
             if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
                 assert username and indexname
-                url = baseurl + username + "/" + indexname
-                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+                clear_index = True
+                break
+        if clear_index:
+            url = baseurl + username + "/" + indexname
+            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
 
 
 

ci_scripts/docker-coredeps/Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/pypa/manylinux1_x86_64
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/
+rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share

ci_scripts/remote_python_packaging.sh

@@ -46,6 +46,6 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
 mkdir -p workspace 
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux1*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
 rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
 rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/run_all.sh

@@ -21,7 +21,8 @@ export DCC_RS_DEV=$(pwd)
 export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
+rm -f python3.5
+ln -s /opt/python/cp35-cp35m/bin/python3.5
 ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
 ln -s /opt/python/cp38-cp38/bin/python3.8

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.28.0"
+version = "1.39.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,10 +20,12 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
+async-std = "1.6.0"
 anyhow = "1.0.28"
 thiserror = "1.0.14"
 
 [features]
-default = ["vendored", "nightly"]
+default = ["vendored"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
+

deltachat_derive/src/lib.rs

@@ -8,36 +8,33 @@ use quote::quote;
 // data.  If this assumption is violated, compiler error will point to
 // generated code, which is not very user-friendly.
 
-#[proc_macro_derive(ToSql)]
-pub fn to_sql_derive(input: TokenStream) -> TokenStream {
+#[proc_macro_derive(Sqlx)]
+pub fn sqlx_derive(input: TokenStream) -> TokenStream {
     let ast: syn::DeriveInput = syn::parse(input).unwrap();
     let name = &ast.ident;
 
     let gen = quote! {
-        impl rusqlite::types::ToSql for #name {
-            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-                let num = *self as i64;
-                let value = rusqlite::types::Value::Integer(num);
-                let output = rusqlite::types::ToSqlOutput::Owned(value);
-                std::result::Result::Ok(output)
-            }
-        }
-    };
-    gen.into()
-}
-
-#[proc_macro_derive(FromSql)]
-pub fn from_sql_derive(input: TokenStream) -> TokenStream {
-    let ast: syn::DeriveInput = syn::parse(input).unwrap();
-    let name = &ast.ident;
-
-    let gen = quote! {
-        impl rusqlite::types::FromSql for #name {
-            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-                let inner = rusqlite::types::FromSql::column_result(col)?;
-                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
+        impl<'q> sqlx::encode::Encode<'q, sqlx::sqlite::Sqlite> for #name {
+            fn encode_by_ref(&self, buf: &mut Vec<sqlx::sqlite::SqliteArgumentValue<'q>>) -> sqlx::encode::IsNull {
+                num_traits::ToPrimitive::to_i32(self).expect("invalid type").encode(buf)
             }
         }
+
+
+        impl<'de> sqlx::decode::Decode<'de, sqlx::sqlite::Sqlite> for #name {
+            fn decode(value: sqlx::sqlite::SqliteValueRef) -> std::result::Result<Self, Box<dyn std::error::Error + 'static + Send + Sync>> {
+                let raw: i32 = sqlx::decode::Decode::decode(value)?;
+
+                Ok(num_traits::FromPrimitive::from_i32(raw).unwrap_or_default())
+            }
+        }
+
+        impl sqlx::types::Type<sqlx::sqlite::Sqlite> for #name {
+            fn type_info() -> sqlx::sqlite::SqliteTypeInfo {
+                <i32 as sqlx::types::Type<_>>::type_info()
+            }
+        }
+
     };
     gen.into()
 }

draft/mailing_list.rst

@@ -1,19 +0,0 @@
-2. Mailing lists are now only detected by the ListId header, because how should DC sort mailing lists if the ListId header is unset and only the precedence header says it's a mailing list (?). We could just hide these emails (with "junk" or "list" precedence), as we did before, if there is a reason to do so. 
-
-   I do not actually understand why such emails were hidden in the first place, though; if there is an automatic answer stating that someone is out of office or if an email could not be delivered (these are the usual reasons why such emails are sent), I would want to know about this as a user.
-   
-  Björn: to the ListId-header: it is already an improvement to use only that and to keep ignoring mails with Precedence-header. historically, i ignored all these mails to reduce noise that is created from them (eg. contacts should not be added to the suggestion list...) and to be able to concentrate on other things first.
-   
-   
-3. for `List-Id: foo <bla>`, bla now is the id and foo is the name. For GitHub and GitLab notifications this is bad because all notifications will go into one chat. Maybe we should instead take the "in-reply-to" header to find out what messages belong together. For normal mailing lists, of course, this will create a second chat if someone does not use the "Reply" button to write to that mailing list.
-
-  Björn: well, having all notifications in one chat is not that bad. i would just follow the guidelines for now, use the ID and not the Name.
-
-
-4. Currently a mailing list is shown as an empty group (ChatType `Group`) ("0 members"). 
-
-   Maybe we should change the ChatType to `Single` because this way, the UI would fit better. Disadvantage: We can't show the different senders of the messages. 
-   
-   Or we introduce a `ChatType` `MailingList`. Very big disadvantage: We would have to adapt all UI project and it would be nice if we could keep the changes within the core.
-
-5. Contacts from mailings lists stay "unknown" now and are not shown in the contacts suggestion list..

draft/mailing_list_current_state.rst

@@ -1,13 +0,0 @@
-1. Mailing lists with Precedence-header and without List-Id are shown as normal messages.
-   
-2. for `List-Id: foo <bla>`, bla now is the id and foo is the name. If foo is not present, bla is taken as the name as well. For GitHub and GitLab notifications all notifications will go into one chat. To distinguish, all subjects are shown in mailing lists.
-
-3. Currently a mailing list is shown as a group with SELF and the List-Id. I could not find any stable way to get a mail address that could be called the "mailing list address". To get this to work, I disabled the may_be_valid_addr check. (to be discussed...)
-
-4. Contacts from mailings lists stay "unknown" and are not shown in the contacts suggestion list.
-
-5. In general, only the From: address is taken as the author. If the domain matches with the List-Id domain (like `deltachat.github.com` and `Hocuri <notifications@notification.github.com>`) then the display name is added to the email address -> `Hocuri - notifications@notification.github.com`. This is not really nice, but as the UIs distinguish bettween contacts only based on the address this was the only way I could find without changing the API. 
-
-   TODO: Prevent the user from sending emails to such contacts (like Hocuri - notifications@notification.github.com)
-
-6. You can't send to mailing lists.

draft/mailing_list_managers.md

@@ -1,71 +0,0 @@
-# Mailing list integration of Delta Chat
-
-A collection of information to help with the integration of mailing lists into Delta Chat.
-
-## Chat name
-
-How should the chat be titled?
-
-It could be taken from the email header `List-Id`:
-
-### Mailman 
-
-* Schema: `${list-description} <${list-local-part}.${list-domain}>`
-* $list-description could be many words, should be truncated. Could also be empty, though.
-
-### Schleuder
-* Schema: `<${list-local-part}.${list-domain}>`
-* No name available.
-* Could be absent: <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L357-358>.
-
-
-### Sympa
-* Schema: `<${list-local-part}.${list-domain}>`
-* No name available.
-* Always present <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/Spindle/TransformOutgoing.pm#L110-L118>, <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/List.pm#L6832-L6833>.
-
-
-## Sender name
-
-What's the name of the person that actually sent the message?
-
-Some MLM change the sender information to avoid problems with DMARC.
-
-### Mailman
-
-* Since v2.1.16 the `From` depends on the list's configuration and possibly on the sender-domain's DMARC-configuration — i.e. messages from the same list can arrive one of the Variants A-D!
-* Documentation of config options:
- * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-544>,
- * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-163>,
- * <https://wiki.list.org/DEV/DMARC>
-* Variant A: `From` is unchanged.
-* Variant B. `From` is mangled like this: `${sender-name} via ${list-name} <${list-addr-spec}>`. The original sender is put into `Reply-To`.
-* Variant C. `From` is mangled like this: `${sender-name} <${encoded-sender-addr-spec}@${list-domain}>`. The original sender is put into `Reply-To`.
-* Variant D: `From` is set to ${list-name-addr}, the originally sent message is included as mime-part.
-* Variant E: `From` is set to ${list-name-addr}, original sender information is removed.
-
-### Schleuder
-
-* Visible only in mime-body (which is possibly encrypted), or not at all.
-* The first `text/plain` mime-part may include the information, as taken from the original message: `From: ${sender-name-addr}`.
-
-### Sympa
-* Depends on the list's configuration.
-* Documentation:
- * <https://sympa-community.github.io/manual/customize/dmarc-protection.html>,
- * <https://sympa-community.github.io/gpldoc/man/sympa.conf.5.html#dmarc-protection>.
-* Variant A: `From` is unchanged.
-* Variant B: `From` is mangled like Mailman Variant B, but the original sender information is put into `X-Original-From`.
-
-
-## Autocrypt
-
-### Mailman
-* Not supported.
-
-### Schleuder
-* A list's key is included in sent messages (as of version 3.5.0) (optional, by default active): <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L349-355>.
-* Incoming keys are not yet looked at (that feature is planned: <https://0xacab.org/schleuder/schleuder/issues/435>).
-
-### Sympa
-* Not supported.

examples/repl/cmdline.rs

@@ -1,7 +1,7 @@
-use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::{bail, ensure};
+use async_std::path::Path;
 use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
@@ -11,7 +11,6 @@ use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
 use deltachat::error::Error;
 use deltachat::imex::*;
-use deltachat::job::*;
 use deltachat::location;
 use deltachat::lot::LotState;
 use deltachat::message::{self, Message, MessageState, MsgId};
@@ -24,78 +23,79 @@ use deltachat::{config, provider};
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
 /// e.g. bitmask 7 triggers actions definded with bits 1, 2 and 4.
-fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
+async fn reset_tables(context: &Context, bits: i32) {
     println!("Resetting tables ({})...", bits);
     if 0 != bits & 1 {
-        sql::execute(context, &context.sql, "DELETE FROM jobs;", params![]).unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM jobs;", paramsx![])
+            .await
+            .unwrap();
         println!("(1) Jobs reset.");
     }
     if 0 != bits & 2 {
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM acpeerstates;",
-            params![],
-        )
-        .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM acpeerstates;", paramsx![])
+            .await
+            .unwrap();
         println!("(2) Peerstates reset.");
     }
     if 0 != bits & 4 {
-        sql::execute(context, &context.sql, "DELETE FROM keypairs;", params![]).unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM keypairs;", paramsx![])
+            .await
+            .unwrap();
         println!("(4) Private keypairs reset.");
     }
     if 0 != bits & 8 {
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM contacts WHERE id>9;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM chats WHERE id>9;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM chats_contacts;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM msgs WHERE id>9;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
-            params![],
-        )
-        .unwrap();
-        sql::execute(context, &context.sql, "DELETE FROM leftgrps;", params![]).unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM contacts WHERE id>9;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM chats WHERE id>9;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM chats_contacts;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM msgs WHERE id>9;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute(
+                "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
+                paramsx![],
+            )
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM leftgrps;", paramsx![])
+            .await
+            .unwrap();
         println!("(8) Rest but server config reset.");
     }
 
-    context.call_cb(Event::MsgsChanged {
+    context.emit_event(Event::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
-
-    1
 }
 
-fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
-    let data = dc_read_file(context, filename)?;
+async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
+    let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false) {
+    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -104,38 +104,29 @@ fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(),
 /// Import a file to the database.
 /// For testing, import a folder with eml-files, a single eml-file, e-mail plus public key and so on.
 /// For normal importing, use imex().
-///
-/// @private @memberof Context
-/// @param context The context as created by dc_context_new().
-/// @param spec The file or directory to import. NULL for the last command.
-/// @return 1=success, 0=error.
-fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
-    if !context.sql.is_open() {
-        error!(context, "Import: Database not opened.");
-        return 0;
-    }
-
-    let mut read_cnt = 0;
+async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
+    let mut read_cnt: usize = 0;
 
     let real_spec: String;
 
-    /* if `spec` is given, remember it for later usage; if it is not given, try to use the last one */
+    // if `spec` is given, remember it for later usage; if it is not given, try to use the last one
     if let Some(spec) = spec {
         real_spec = spec.to_string();
         context
-            .sql
+            .sql()
             .set_raw_config(context, "import_spec", Some(&real_spec))
+            .await
             .unwrap();
     } else {
-        let rs = context.sql.get_raw_config(context, "import_spec");
+        let rs = context.sql().get_raw_config("import_spec").await;
         if rs.is_none() {
             error!(context, "Import: No file or folder given.");
-            return 0;
+            return false;
         }
         real_spec = rs.unwrap();
     }
     if let Some(suffix) = dc_get_filesuffix_lc(&real_spec) {
-        if suffix == "eml" && dc_poke_eml_file(context, &real_spec).is_ok() {
+        if suffix == "eml" && poke_eml_file(context, &real_spec).await.is_ok() {
             read_cnt += 1
         }
     } else {
@@ -144,7 +135,7 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
         let dir = std::fs::read_dir(dir_name);
         if dir.is_err() {
             error!(context, "Import: Cannot open directory \"{}\".", &real_spec,);
-            return 0;
+            return false;
         } else {
             let dir = dir.unwrap();
             for entry in dir {
@@ -157,7 +148,7 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
                     println!("Import: {}", path_plus_name);
-                    if dc_poke_eml_file(context, path_plus_name).is_ok() {
+                    if poke_eml_file(context, path_plus_name).await.is_ok() {
                         read_cnt += 1
                     }
                 }
@@ -166,16 +157,19 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.call_cb(Event::MsgsChanged {
+        context.emit_event(Event::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
     }
-    1
+    true
 }
 
-fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
-    let contact = Contact::get_by_id(context, msg.get_from_id()).expect("invalid contact");
+async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
+    let contact = Contact::get_by_id(context, msg.get_from_id())
+        .await
+        .expect("invalid contact");
+
     let contact_name = contact.get_name();
     let contact_id = contact.get_id();
 
@@ -218,7 +212,7 @@ fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     );
 }
 
-fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
+async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id.is_daymarker() {
@@ -234,8 +228,8 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
                 );
                 lines_out += 1
             }
-            let msg = Message::load_from_db(context, msg_id)?;
-            log_msg(context, "", &msg);
+            let msg = Message::load_from_db(context, msg_id).await?;
+            log_msg(context, "", &msg).await;
         }
     }
     if lines_out > 0 {
@@ -246,18 +240,18 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     Ok(())
 }
 
-fn log_contactlist(context: &Context, contacts: &Vec<u32>) {
-    let mut contacts = contacts.clone();
+async fn log_contactlist(context: &Context, contacts: &[u32]) {
+    let mut contacts = contacts.to_vec();
     if !contacts.contains(&1) {
         contacts.push(1);
     }
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, contact_id) {
+        if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
             let name = contact.get_name();
             let addr = contact.get_addr();
-            let verified_state = contact.is_verified(context);
+            let verified_state = contact.is_verified(context).await;
             let verified_str = if VerifiedStatus::Unverified != verified_state {
                 if verified_state == VerifiedStatus::BidirectVerified {
                     " √√"
@@ -281,8 +275,8 @@ fn log_contactlist(context: &Context, contacts: &Vec<u32>) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, &context.sql, &addr);
-            if peerstate.is_some() && contact_id != 1 as libc::c_uint {
+            let peerstate = Peerstate::from_addr(context, &addr).await;
+            if peerstate.is_ok() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
                     peerstate.as_ref().unwrap().prefer_encrypt
@@ -298,10 +292,9 @@ fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
-    let chat_id = *context.cmdline_sel_chat_id.read().unwrap();
+pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<(), Error> {
     let mut sel_chat = if !chat_id.is_unset() {
-        Chat::load_from_db(context, chat_id).ok()
+        Chat::load_from_db(&context, *chat_id).await.ok()
     } else {
         None
     };
@@ -342,7 +335,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  configure\n\
                  connect\n\
                  disconnect\n\
-                 interrupt\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -405,7 +397,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  ============================================="
             ),
         },
-        "initiate-key-transfer" => match initiate_key_transfer(context) {
+        "initiate-key-transfer" => match initiate_key_transfer(&context).await {
             Ok(setup_code) => println!(
                 "Setup code for the transferred setup message: {}",
                 setup_code,
@@ -415,9 +407,9 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "get-setupcodebegin" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let msg_id: MsgId = MsgId::new(arg1.parse()?);
-            let msg = Message::load_from_db(context, msg_id)?;
+            let msg = Message::load_from_db(&context, msg_id).await?;
             if msg.is_setupmessage() {
-                let setupcodebegin = msg.get_setupcodebegin(context);
+                let setupcodebegin = msg.get_setupcodebegin(&context).await;
                 println!(
                     "The setup code for setup message {} starts with: {}",
                     msg_id,
@@ -432,29 +424,29 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(context, MsgId::new(arg1.parse()?), &arg2)?;
+            continue_key_transfer(&context, MsgId::new(arg1.parse()?), &arg2).await?;
         }
         "has-backup" => {
-            has_backup(context, blobdir)?;
+            has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            imex(context, ImexMode::ExportBackup, Some(blobdir));
+            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(context, ImexMode::ImportBackup, Some(arg1));
+            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
-            imex(context, ImexMode::ExportSelfKeys, Some(blobdir));
+            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
         }
         "import-keys" => {
-            imex(context, ImexMode::ImportSelfKeys, Some(blobdir));
+            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
         }
         "export-setup" => {
-            let setup_code = create_setup_code(context);
+            let setup_code = create_setup_code(&context);
             let file_name = blobdir.join("autocrypt-setup-message.html");
-            let file_content = render_setup_file(context, &setup_code)?;
-            std::fs::write(&file_name, file_content)?;
+            let file_content = render_setup_file(&context, &setup_code).await?;
+            async_std::fs::write(&file_name, file_content).await?;
             println!(
                 "Setup message written to: {}\nSetup code: {}",
                 file_name.display(),
@@ -462,49 +454,51 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             );
         }
         "poke" => {
-            ensure!(0 != poke_spec(context, Some(arg1)), "Poke failed");
+            ensure!(poke_spec(&context, Some(arg1)).await, "Poke failed");
         }
         "reset" => {
             ensure!(!arg1.is_empty(), "Argument <bits> missing: 1=jobs, 2=peerstates, 4=private keys, 8=rest but server config");
             let bits: i32 = arg1.parse()?;
             ensure!(bits < 16, "<bits> must be lower than 16.");
-            ensure!(0 != dc_reset_tables(context, bits), "Reset failed");
+            reset_tables(&context, bits).await;
         }
         "stop" => {
-            context.stop_ongoing();
+            context.stop_ongoing().await;
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
             let key = config::Config::from_str(&arg1)?;
             let value = if arg2.is_empty() { None } else { Some(arg2) };
-            context.set_config(key, value)?;
+            context.set_config(key, value).await?;
         }
         "get" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
             let key = config::Config::from_str(&arg1)?;
-            let val = context.get_config(key);
+            let val = context.get_config(key).await;
             println!("{}={:?}", key, val);
         }
         "info" => {
-            println!("{:#?}", context.get_info());
-        }
-        "interrupt" => {
-            interrupt_inbox_idle(context);
+            println!("{:#?}", context.get_info().await);
         }
         "maybenetwork" => {
-            maybe_network(context);
+            context.maybe_network().await;
         }
         "housekeeping" => {
-            sql::housekeeping(context);
+            sql::housekeeping(&context).await?;
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
+            let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
-                context,
+                &context,
                 listflags,
                 if arg1.is_empty() { None } else { Some(arg1) },
                 None,
-            )?;
+            )
+            .await?;
+            let time_needed = std::time::SystemTime::now()
+                .duration_since(time_start)
+                .unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -513,20 +507,20 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 );
 
                 for i in (0..cnt).rev() {
-                    let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
                         "{}#{}: {} [{} fresh] {}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat.get_id().get_fresh_msg_cnt(context),
+                        chat.get_id().get_fresh_msg_cnt(&context).await,
                         match chat.visibility {
                             ChatVisibility::Normal => "",
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
                     );
-                    let lot = chatlist.get_summary(context, i, Some(&chat));
+                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
@@ -559,31 +553,32 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(context, ChatId::new(0)) {
+            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
+            println!("{:?} to create this list", time_needed);
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
                 bail!("Argument [chat-id] is missing.");
             }
             if !arg1.is_empty() {
-                let chat_id = ChatId::new(arg1.parse()?);
-                println!("Selecting chat {}", chat_id);
-                sel_chat = Some(Chat::load_from_db(context, chat_id)?);
-                *context.cmdline_sel_chat_id.write().unwrap() = chat_id;
+                let id = ChatId::new(arg1.parse()?);
+                println!("Selecting chat {}", id);
+                sel_chat = Some(Chat::load_from_db(&context, id).await?);
+                *chat_id = id;
             }
 
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, None);
-            let members = chat::get_chat_contacts(context, sel_chat.id);
+            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await?;
+            let members = chat::get_chat_contacts(&context, sel_chat.id).await?;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
             } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
-                let contact = Contact::get_by_id(context, members[0])?;
+                let contact = Contact::get_by_id(&context, members[0]).await?;
                 contact.get_addr().to_string()
             } else {
                 format!("{} member(s)", members.len())
@@ -599,7 +594,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 } else {
                     ""
                 },
-                match sel_chat.get_profile_image(context) {
+                match sel_chat.get_profile_image(&context).await? {
                     Some(icon) => match icon.to_str() {
                         Some(icon) => format!(" Icon: {}", icon),
                         _ => " Icon: Err".to_string(),
@@ -607,38 +602,42 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     _ => "".to_string(),
                 },
             );
-            log_msglist(context, &msglist)?;
-            if let Some(draft) = sel_chat.get_id().get_draft(context)? {
-                log_msg(context, "Draft", &draft);
+            log_msglist(&context, &msglist).await?;
+            if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
+                log_msg(&context, "Draft", &draft).await;
             }
 
-            println!("{} messages.", sel_chat.get_id().get_msg_cnt(context));
-            chat::marknoticed_chat(context, sel_chat.get_id())?;
+            println!(
+                "{} messages.",
+                sel_chat.get_id().get_msg_cnt(&context).await
+            );
+            chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id: libc::c_int = arg1.parse()?;
-            let chat_id = chat::create_by_contact_id(context, contact_id as u32)?;
+            let chat_id = chat::create_by_contact_id(&context, contact_id as u32).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
         "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
             let msg_id = MsgId::new(arg1.parse()?);
-            let chat_id = chat::create_by_msg_id(context, msg_id)?;
-            let chat = Chat::load_from_db(context, chat_id)?;
+            let chat_id = chat::create_by_msg_id(&context, msg_id).await?;
+            let chat = Chat::load_from_db(&context, chat_id).await?;
 
             println!("{}#{} created successfully.", chat_prefix(&chat), chat_id,);
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(context, VerifiedStatus::Unverified, arg1)?;
+            let chat_id =
+                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
         "createverified" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(context, VerifiedStatus::Verified, arg1)?;
+            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
 
             println!("VerifiedGroup#{} created successfully.", chat_id);
         }
@@ -648,10 +647,12 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             let contact_id_0: libc::c_int = arg1.parse()?;
             if chat::add_contact_to_chat(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 contact_id_0 as u32,
-            ) {
+            )
+            .await
+            {
                 println!("Contact added to chat.");
             } else {
                 bail!("Cannot add contact to chat.");
@@ -662,17 +663,18 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id_1: libc::c_int = arg1.parse()?;
             chat::remove_contact_from_chat(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 contact_id_1 as u32,
-            )?;
+            )
+            .await?;
 
             println!("Contact added to chat.");
         }
         "groupname" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            chat::set_chat_name(context, sel_chat.as_ref().unwrap().get_id(), arg1)?;
+            chat::set_chat_name(&context, sel_chat.as_ref().unwrap().get_id(), arg1).await?;
 
             println!("Chat name set");
         }
@@ -680,24 +682,27 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <image> missing.");
 
-            chat::set_chat_profile_image(context, sel_chat.as_ref().unwrap().get_id(), arg1)?;
+            chat::set_chat_profile_image(&context, sel_chat.as_ref().unwrap().get_id(), arg1)
+                .await?;
 
             println!("Chat image set");
         }
         "chatinfo" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contacts = chat::get_chat_contacts(context, sel_chat.as_ref().unwrap().get_id());
+            let contacts =
+                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
             println!("Memberlist:");
 
-            log_contactlist(context, &contacts);
+            log_contactlist(&context, &contacts).await;
             println!(
                 "{} contacts\nLocation streaming: {}",
                 contacts.len(),
                 location::is_sending_locations_to_chat(
-                    context,
+                    &context,
                     sel_chat.as_ref().unwrap().get_id()
-                ),
+                )
+                .await,
             );
         }
         "getlocations" => {
@@ -705,12 +710,13 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             let contact_id = arg1.parse().unwrap_or_default();
             let locations = location::get_range(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 contact_id,
                 0,
                 0,
-            );
+            )
+            .await;
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
@@ -736,7 +742,12 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             ensure!(!arg1.is_empty(), "No timeout given.");
 
             let seconds = arg1.parse()?;
-            location::send_locations_to_chat(context, sel_chat.as_ref().unwrap().get_id(), seconds);
+            location::send_locations_to_chat(
+                &context,
+                sel_chat.as_ref().unwrap().get_id(),
+                seconds,
+            )
+            .await;
             println!(
                 "Locations will be sent to Chat#{} for {} seconds. Use 'setlocation <lat> <lng>' to play around.",
                 sel_chat.as_ref().unwrap().get_id(),
@@ -751,7 +762,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             let latitude = arg1.parse()?;
             let longitude = arg2.parse()?;
 
-            let continue_streaming = location::set(context, latitude, longitude, 0.);
+            let continue_streaming = location::set(&context, latitude, longitude, 0.).await?;
             if continue_streaming {
                 println!("Success, streaming should be continued.");
             } else {
@@ -759,7 +770,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             }
         }
         "dellocations" => {
-            location::delete_all(context)?;
+            location::delete_all(&context).await?;
         }
         "send" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -767,11 +778,11 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             let msg = format!("{} {}", arg1, arg2);
 
-            chat::send_text_msg(context, sel_chat.as_ref().unwrap().get_id(), msg)?;
+            chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), msg).await?;
         }
         "sendempty" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
-            chat::send_text_msg(context, sel_chat.as_ref().unwrap().get_id(), "".into())?;
+            chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), "".into()).await?;
         }
         "sendimage" | "sendfile" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -786,7 +797,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             if !arg2.is_empty() {
                 msg.set_text(Some(arg2.to_string()));
             }
-            chat::send_msg(context, sel_chat.as_ref().unwrap().get_id(), &mut msg)?;
+            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
@@ -797,9 +808,9 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 ChatId::new(0)
             };
 
-            let msglist = context.search_msgs(chat, arg1);
+            let msglist = context.search_msgs(chat, arg1).await;
 
-            log_msglist(context, &msglist)?;
+            log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
         }
         "draft" => {
@@ -812,10 +823,16 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     .as_ref()
                     .unwrap()
                     .get_id()
-                    .set_draft(context, Some(&mut draft));
+                    .set_draft(&context, Some(&mut draft))
+                    .await;
                 println!("Draft saved.");
             } else {
-                sel_chat.as_ref().unwrap().get_id().set_draft(context, None);
+                sel_chat
+                    .as_ref()
+                    .unwrap()
+                    .get_id()
+                    .set_draft(&context, None)
+                    .await;
                 println!("Draft deleted.");
             }
         }
@@ -826,21 +843,22 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             );
             let mut msg = Message::new(Viewtype::Text);
             msg.set_text(Some(arg1.to_string()));
-            chat::add_device_msg(context, None, Some(&mut msg))?;
+            chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
         "updatedevicechats" => {
-            context.update_device_chats()?;
+            context.update_device_chats().await?;
         }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let images = chat::get_chat_media(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 Viewtype::Image,
                 Viewtype::Gif,
                 Viewtype::Video,
-            );
+            )
+            .await?;
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
@@ -849,36 +867,38 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     print!(", {}", data);
                 }
             }
-            print!("\n");
+            println!();
         }
         "archive" | "unarchive" | "pin" | "unpin" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.set_visibility(
-                context,
-                match arg0 {
-                    "archive" => ChatVisibility::Archived,
-                    "unarchive" | "unpin" => ChatVisibility::Normal,
-                    "pin" => ChatVisibility::Pinned,
-                    _ => panic!("Unexpected command (This should never happen)"),
-                },
-            )?;
+            chat_id
+                .set_visibility(
+                    &context,
+                    match arg0 {
+                        "archive" => ChatVisibility::Archived,
+                        "unarchive" | "unpin" => ChatVisibility::Normal,
+                        "pin" => ChatVisibility::Pinned,
+                        _ => panic!("Unexpected command (This should never happen)"),
+                    },
+                )
+                .await?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.delete(context)?;
+            chat_id.delete(&context).await?;
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
-            let res = message::get_msg_info(context, id);
+            let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
         "listfresh" => {
-            let msglist = context.get_fresh_msgs();
+            let msglist = context.get_fresh_msgs().await;
 
-            log_msglist(context, &msglist)?;
+            log_msglist(&context, &msglist).await?;
             print!("{} fresh messages.", msglist.len());
         }
         "forward" => {
@@ -890,37 +910,38 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             let mut msg_ids = [MsgId::new(0); 1];
             let chat_id = ChatId::new(arg2.parse()?);
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            chat::forward_msgs(context, &msg_ids, chat_id)?;
+            chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [MsgId::new(0); 1];
+            let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(context, &msg_ids);
+            message::markseen_msgs(&context, msg_ids).await;
         }
         "star" | "unstar" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [MsgId::new(0); 1];
+            let mut msg_ids = vec![MsgId::new(0); 1];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::star_msgs(context, &msg_ids, arg0 == "star");
+            message::star_msgs(&context, msg_ids, arg0 == "star").await;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
             ids[0] = MsgId::new(arg1.parse()?);
-            message::delete_msgs(context, &ids);
+            message::delete_msgs(&context, &ids).await;
         }
         "listcontacts" | "contacts" | "listverified" => {
             let contacts = Contact::get_all(
-                context,
+                &context,
                 if arg0 == "listverified" {
                     0x1 | 0x2
                 } else {
                     0x2
                 },
                 Some(arg1),
-            )?;
-            log_contactlist(context, &contacts);
+            )
+            .await?;
+            log_contactlist(&context, &contacts).await;
             println!("{} contacts.", contacts.len());
         }
         "addcontact" => {
@@ -928,30 +949,30 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(context, book)?;
+                Contact::add_address_book(&context, book).await?;
             } else {
-                Contact::create(context, "", arg1)?;
+                Contact::create(&context, "", arg1).await?;
             }
         }
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
             let contact_id = arg1.parse()?;
-            let contact = Contact::get_by_id(context, contact_id)?;
+            let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
             let mut res = format!(
                 "Contact info for: {}:\nIcon: {}\n",
                 name_n_addr,
-                match contact.get_profile_image(context) {
+                match contact.get_profile_image(&context).await {
                     Some(image) => image.to_str().unwrap().to_string(),
                     None => "NoIcon".to_string(),
                 }
             );
 
-            res += &Contact::get_encrinfo(context, contact_id)?;
+            res += &Contact::get_encrinfo(&context, contact_id).await?;
 
-            let chatlist = Chatlist::try_load(context, 0, None, Some(contact_id))?;
+            let chatlist = Chatlist::try_load(&context, 0, None, Some(contact_id)).await?;
             let chatlist_cnt = chatlist.len();
             if chatlist_cnt > 0 {
                 res += &format!(
@@ -962,7 +983,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     if 0 != i {
                         res += ", ";
                     }
-                    let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     res += &format!("{}#{}", chat_prefix(&chat), chat.get_id());
                 }
             }
@@ -971,11 +992,11 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            Contact::delete(context, arg1.parse()?)?;
+            Contact::delete(&context, arg1.parse()?).await?;
         }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            let res = check_qr(context, arg1);
+            let res = check_qr(&context, arg1).await;
             println!(
                 "state={}, id={}, text1={:?}, text2={:?}",
                 res.get_state(),
@@ -986,7 +1007,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            match set_config_from_qr(context, arg1) {
+            match set_config_from_qr(&context, arg1).await {
                 Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
                 Err(err) => println!("Cannot set config from QR code: {:?}", err),
             }
@@ -1014,7 +1035,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
         //     let event = arg1.parse()?;
         //     let event = Event::from_u32(event).ok_or(format_err!("Event::from_u32({})", event))?;
-        //     let r = context.call_cb(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
+        //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",
         //         event, event as usize, r as libc::c_int,
@@ -1023,7 +1044,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 
-            if let Ok(buf) = dc_read_file(context, &arg1) {
+            if let Ok(buf) = dc_read_file(&context, &arg1).await {
                 let (width, height) = dc_get_filemeta(&buf)?;
                 println!("width={}, height={}", width, height);
             } else {
@@ -1033,8 +1054,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "estimatedeletion" => {
             ensure!(!arg1.is_empty(), "Argument <seconds> missing");
             let seconds = arg1.parse()?;
-            let device_cnt = message::estimate_deletion_cnt(context, false, seconds)?;
-            let server_cnt = message::estimate_deletion_cnt(context, true, seconds)?;
+            let device_cnt = message::estimate_deletion_cnt(&context, false, seconds).await?;
+            let server_cnt = message::estimate_deletion_cnt(&context, true, seconds).await?;
             println!(
                 "estimated count of messages older than {} seconds:\non device: {}\non server: {}",
                 seconds, device_cnt, server_cnt
@@ -1043,7 +1064,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "emptyserver" => {
             ensure!(!arg1.is_empty(), "Argument <flags> missing");
 
-            message::dc_empty_server(context, arg1.parse()?);
+            message::dc_empty_server(&context, arg1.parse()?).await;
         }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),

examples/repl/main.rs

@@ -6,26 +6,21 @@
 
 #[macro_use]
 extern crate deltachat;
-#[macro_use]
-extern crate lazy_static;
-#[macro_use]
-extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
-use std::path::Path;
 use std::process::Command;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
 
+use ansi_term::Color;
 use anyhow::{bail, Error};
+use async_std::path::Path;
 use deltachat::chat::ChatId;
 use deltachat::config;
 use deltachat::context::*;
-use deltachat::job::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
 use deltachat::Event;
+use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
@@ -38,179 +33,83 @@ use rustyline::{
 mod cmdline;
 use self::cmdline::*;
 
-// Event Handler
-
-fn receive_event(_context: &Context, event: Event) {
+/// Event Handler
+fn receive_event(event: Event) {
+    let yellow = Color::Yellow.normal();
     match event {
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
-            println!("{}", msg);
+            info!("{}", msg);
         }
         Event::SmtpConnected(msg) => {
-            println!("[DC_EVENT_SMTP_CONNECTED] {}", msg);
+            info!("[SMTP_CONNECTED] {}", msg);
         }
         Event::ImapConnected(msg) => {
-            println!("[DC_EVENT_IMAP_CONNECTED] {}", msg);
+            info!("[IMAP_CONNECTED] {}", msg);
         }
         Event::SmtpMessageSent(msg) => {
-            println!("[DC_EVENT_SMTP_MESSAGE_SENT] {}", msg);
+            info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
         Event::Warning(msg) => {
-            println!("[Warning] {}", msg);
+            warn!("{}", msg);
         }
         Event::Error(msg) => {
-            println!("\x1b[31m[DC_EVENT_ERROR] {}\x1b[0m", msg);
+            error!("{}", msg);
         }
         Event::ErrorNetwork(msg) => {
-            println!("\x1b[31m[DC_EVENT_ERROR_NETWORK] msg={}\x1b[0m", msg);
+            error!("[NETWORK] msg={}", msg);
         }
         Event::ErrorSelfNotInGroup(msg) => {
-            println!("\x1b[31m[DC_EVENT_ERROR_SELF_NOT_IN_GROUP] {}\x1b[0m", msg);
+            error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
         Event::MsgsChanged { chat_id, msg_id } => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_MSGS_CHANGED(chat_id={}, msg_id={})}}\n\x1b[0m",
-                chat_id, msg_id,
+            info!(
+                "{}",
+                yellow.paint(format!(
+                    "Received MSGS_CHANGED(chat_id={}, msg_id={})",
+                    chat_id, msg_id,
+                ))
             );
         }
         Event::ContactsChanged(_) => {
-            print!("\x1b[33m{{Received DC_EVENT_CONTACTS_CHANGED()}}\n\x1b[0m");
+            info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
         Event::LocationChanged(contact) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_LOCATION_CHANGED(contact={:?})}}\n\x1b[0m",
-                contact,
+            info!(
+                "{}",
+                yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
         Event::ConfigureProgress(progress) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_CONFIGURE_PROGRESS({} ‰)}}\n\x1b[0m",
-                progress,
+            info!(
+                "{}",
+                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
             );
         }
         Event::ImexProgress(progress) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_IMEX_PROGRESS({} ‰)}}\n\x1b[0m",
-                progress,
+            info!(
+                "{}",
+                yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
         Event::ImexFileWritten(file) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_IMEX_FILE_WRITTEN({})}}\n\x1b[0m",
-                file.display()
+            info!(
+                "{}",
+                yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
         Event::ChatModified(chat) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_CHAT_MODIFIED({})}}\n\x1b[0m",
-                chat
+            info!(
+                "{}",
+                yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
             );
         }
         _ => {
-            print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
+            info!("Received {:?}", event);
         }
     }
 }
 
-// Threads for waiting for messages and for jobs
-
-lazy_static! {
-    static ref HANDLE: Arc<Mutex<Option<Handle>>> = Arc::new(Mutex::new(None));
-    static ref IS_RUNNING: AtomicBool = AtomicBool::new(true);
-}
-
-struct Handle {
-    handle_imap: Option<std::thread::JoinHandle<()>>,
-    handle_mvbox: Option<std::thread::JoinHandle<()>>,
-    handle_sentbox: Option<std::thread::JoinHandle<()>>,
-    handle_smtp: Option<std::thread::JoinHandle<()>>,
-}
-
-macro_rules! while_running {
-    ($code:block) => {
-        if IS_RUNNING.load(Ordering::Relaxed) {
-            $code
-        } else {
-            break;
-        }
-    };
-}
-
-fn start_threads(c: Arc<RwLock<Context>>) {
-    if HANDLE.clone().lock().unwrap().is_some() {
-        return;
-    }
-
-    println!("Starting threads");
-    IS_RUNNING.store(true, Ordering::Relaxed);
-
-    let ctx = c.clone();
-    let handle_imap = std::thread::spawn(move || loop {
-        while_running!({
-            perform_inbox_jobs(&ctx.read().unwrap());
-            perform_inbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                let context = ctx.read().unwrap();
-                perform_inbox_idle(&context);
-            });
-        });
-    });
-
-    let ctx = c.clone();
-    let handle_mvbox = std::thread::spawn(move || loop {
-        while_running!({
-            perform_mvbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                perform_mvbox_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    let ctx = c.clone();
-    let handle_sentbox = std::thread::spawn(move || loop {
-        while_running!({
-            perform_sentbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                perform_sentbox_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    let ctx = c;
-    let handle_smtp = std::thread::spawn(move || loop {
-        while_running!({
-            perform_smtp_jobs(&ctx.read().unwrap());
-            while_running!({
-                perform_smtp_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    *HANDLE.clone().lock().unwrap() = Some(Handle {
-        handle_imap: Some(handle_imap),
-        handle_mvbox: Some(handle_mvbox),
-        handle_sentbox: Some(handle_sentbox),
-        handle_smtp: Some(handle_smtp),
-    });
-}
-
-fn stop_threads(context: &Context) {
-    if let Some(ref mut handle) = *HANDLE.clone().lock().unwrap() {
-        println!("Stopping threads");
-        IS_RUNNING.store(false, Ordering::Relaxed);
-
-        interrupt_inbox_idle(context);
-        interrupt_mvbox_idle(context);
-        interrupt_sentbox_idle(context);
-        interrupt_smtp_idle(context);
-
-        handle.handle_imap.take().unwrap().join().unwrap();
-        handle.handle_mvbox.take().unwrap().join().unwrap();
-        handle.handle_sentbox.take().unwrap().join().unwrap();
-        handle.handle_smtp.take().unwrap().join().unwrap();
-    }
-}
-
 // === The main loop
 
 struct DcHelper {
@@ -247,10 +146,8 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 11] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
-    "open",
-    "close",
     "set",
     "get",
     "oauth2",
@@ -369,69 +266,81 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-fn main_0(args: Vec<String>) -> Result<(), Error> {
+async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(
-        Box::new(receive_event),
-        "CLI".into(),
-        Path::new(&args[1]).to_path_buf(),
-    )?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
+
+    let events = context.get_event_emitter();
+    async_std::task::spawn(async move {
+        while let Some(event) = events.recv().await {
+            receive_event(event);
+        }
+    });
 
     println!("Delta Chat Core is awaiting your commands.");
 
-    let ctx = Arc::new(RwLock::new(context));
-
     let config = Config::builder()
         .history_ignore_space(true)
         .completion_type(CompletionType::List)
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let h = DcHelper {
-        completer: FilenameCompleter::new(),
-        highlighter: MatchingBracketHighlighter::new(),
-        hinter: HistoryHinter {},
-    };
-    let mut rl = Editor::with_config(config);
-    rl.set_helper(Some(h));
-    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
-    if rl.load_history(".dc-history.txt").is_err() {
-        println!("No previous history.");
-    }
+    let mut selected_chat = ChatId::default();
+    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let input_loop = async_std::task::spawn_blocking(move || {
+        let h = DcHelper {
+            completer: FilenameCompleter::new(),
+            highlighter: MatchingBracketHighlighter::new(),
+            hinter: HistoryHinter {},
+        };
+        let mut rl = Editor::with_config(config);
+        rl.set_helper(Some(h));
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        if rl.load_history(".dc-history.txt").is_err() {
+            println!("No previous history.");
+        }
 
-    loop {
-        let p = "> ";
-        let readline = rl.readline(&p);
-        match readline {
-            Ok(line) => {
-                // TODO: ignore "set mail_pw"
-                rl.add_history_entry(line.as_str());
-                let ctx = ctx.clone();
-                match handle_cmd(line.trim(), ctx) {
-                    Ok(ExitResult::Continue) => {}
-                    Ok(ExitResult::Exit) => break,
-                    Err(err) => println!("Error: {}", err),
+        loop {
+            let p = "> ";
+            let readline = rl.readline(&p);
+
+            match readline {
+                Ok(line) => {
+                    // TODO: ignore "set mail_pw"
+                    rl.add_history_entry(line.as_str());
+                    async_std::task::block_on(reader_s.send(line));
+                }
+                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                    println!("Exiting...");
+                    drop(reader_s);
+                    break;
+                }
+                Err(err) => {
+                    println!("Error: {}", err);
+                    drop(reader_s);
+                    break;
                 }
             }
-            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-                println!("Exiting...");
-                break;
-            }
-            Err(err) => {
-                println!("Error: {}", err);
-                break;
-            }
+        }
+
+        rl.save_history(".dc-history.txt")?;
+        println!("history saved");
+        Ok::<_, Error>(())
+    });
+
+    while let Ok(line) = reader_r.recv().await {
+        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
+            Ok(ExitResult::Continue) => {}
+            Ok(ExitResult::Exit) => break,
+            Err(err) => println!("Error: {}", err),
         }
     }
-    rl.save_history(".dc-history.txt")?;
-    println!("history saved");
-    {
-        stop_threads(&ctx.read().unwrap());
-    }
+    context.stop_io().await;
+    input_loop.await?;
 
     Ok(())
 }
@@ -442,43 +351,29 @@ enum ExitResult {
     Exit,
 }
 
-fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
+async fn handle_cmd(
+    line: &str,
+    ctx: Context,
+    selected_chat: &mut ChatId,
+) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
 
     match arg0 {
         "connect" => {
-            start_threads(ctx);
+            ctx.start_io().await;
         }
         "disconnect" => {
-            stop_threads(&ctx.read().unwrap());
-        }
-        "smtp-jobs" => {
-            if HANDLE.clone().lock().unwrap().is_some() {
-                println!("smtp-jobs are already running in a thread.",);
-            } else {
-                perform_smtp_jobs(&ctx.read().unwrap());
-            }
-        }
-        "imap-jobs" => {
-            if HANDLE.clone().lock().unwrap().is_some() {
-                println!("inbox-jobs are already running in a thread.");
-            } else {
-                perform_inbox_jobs(&ctx.read().unwrap());
-            }
+            ctx.stop_io().await;
         }
         "configure" => {
-            start_threads(ctx.clone());
-            ctx.read().unwrap().configure();
+            ctx.configure().await?;
         }
         "oauth2" => {
-            if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
-                let oauth2_url = dc_get_oauth2_url(
-                    &ctx.read().unwrap(),
-                    &addr,
-                    "chat.delta:/com.b44t.messenger",
-                );
+            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
+                let oauth2_url =
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -493,11 +388,10 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
             print!("\x1b[1;1H\x1b[2J");
         }
         "getqr" | "getbadqr" => {
-            start_threads(ctx.clone());
-            if let Some(mut qr) = dc_get_securejoin_qr(
-                &ctx.read().unwrap(),
-                ChatId::new(arg1.parse().unwrap_or_default()),
-            ) {
+            ctx.start_io().await;
+            if let Some(mut qr) =
+                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
+            {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -513,23 +407,23 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
             }
         }
         "joinqr" => {
-            start_threads(ctx.clone());
+            ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx.read().unwrap(), arg1);
+                dc_join_securejoin(&ctx, arg1).await;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),
-        _ => dc_cmdline(&ctx.read().unwrap(), line)?,
+        _ => cmdline(ctx.clone(), line, selected_chat).await?,
     }
 
     Ok(ExitResult::Continue)
 }
 
-pub fn main() -> Result<(), Error> {
+fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
-    let args: Vec<String> = std::env::args().collect();
-    main_0(args)?;
+    let args = std::env::args().collect();
+    async_std::task::block_on(async move { start(args).await })?;
 
     Ok(())
 }

examples/simple.rs

@@ -1,7 +1,3 @@
-extern crate deltachat;
-
-use std::sync::{Arc, RwLock};
-use std::{thread, time};
 use tempfile::tempdir;
 
 use deltachat::chat;
@@ -9,103 +5,96 @@ use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::job::{
-    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
-    perform_smtp_jobs,
-};
+use deltachat::message::Message;
 use deltachat::Event;
 
-fn cb(_ctx: &Context, event: Event) {
-    print!("[{:?}]", event);
-
+fn cb(event: Event) {
     match event {
         Event::ConfigureProgress(progress) => {
-            println!("  progress: {}", progress);
+            log::info!("progress: {}", progress);
         }
-        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            println!("  {}", msg);
+        Event::Info(msg) => {
+            log::info!("{}", msg);
         }
-        _ => {
-            println!();
+        Event::Warning(msg) => {
+            log::warn!("{}", msg);
+        }
+        Event::Error(msg) | Event::ErrorNetwork(msg) => {
+            log::error!("{}", msg);
+        }
+        event => {
+            log::info!("{:?}", event);
         }
     }
 }
 
-fn main() {
+/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
+#[async_std::main]
+async fn main() {
+    pretty_env_logger::try_init_timed().ok();
+
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    println!("creating database {:?}", dbfile);
-    let ctx =
-        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
-    let running = Arc::new(RwLock::new(true));
-    let info = ctx.get_info();
-    let duration = time::Duration::from_millis(4000);
-    println!("info: {:#?}", info);
+    log::info!("creating database {:?}", dbfile);
+    let ctx = Context::new("FakeOs".into(), dbfile.into())
+        .await
+        .expect("Failed to create context");
+    let info = ctx.get_info().await;
+    log::info!("info: {:#?}", info);
 
-    let ctx = Arc::new(ctx);
-    let ctx1 = ctx.clone();
-    let r1 = running.clone();
-    let t1 = thread::spawn(move || {
-        while *r1.read().unwrap() {
-            perform_inbox_jobs(&ctx1);
-            if *r1.read().unwrap() {
-                perform_inbox_fetch(&ctx1);
-
-                if *r1.read().unwrap() {
-                    perform_inbox_idle(&ctx1);
-                }
-            }
+    let events = ctx.get_event_emitter();
+    let events_spawn = async_std::task::spawn(async move {
+        while let Some(event) = events.recv().await {
+            cb(event);
         }
     });
 
-    let ctx1 = ctx.clone();
-    let r1 = running.clone();
-    let t2 = thread::spawn(move || {
-        while *r1.read().unwrap() {
-            perform_smtp_jobs(&ctx1);
-            if *r1.read().unwrap() {
-                perform_smtp_idle(&ctx1);
-            }
-        }
-    });
-
-    println!("configuring");
+    log::info!("configuring");
     let args = std::env::args().collect::<Vec<String>>();
-    assert_eq!(args.len(), 2, "missing password");
-    let pw = args[1].clone();
-    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
+    assert_eq!(args.len(), 3, "requires email password");
+    let email = args[1].clone();
+    let pw = args[2].clone();
+    ctx.set_config(config::Config::Addr, Some(&email))
+        .await
+        .unwrap();
+    ctx.set_config(config::Config::MailPw, Some(&pw))
+        .await
         .unwrap();
-    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-    ctx.configure();
 
-    thread::sleep(duration);
+    ctx.configure().await.unwrap();
 
-    println!("sending a message");
-    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
-    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
-    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
+    log::info!("------ RUN ------");
+    ctx.start_io().await;
+    log::info!("--- SENDING A MESSAGE ---");
 
-    println!("fetching chats..");
-    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
+    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
+        .await
+        .unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
-    for i in 0..chats.len() {
-        let summary = chats.get_summary(&ctx, 0, None);
-        let text1 = summary.get_text1();
-        let text2 = summary.get_text2();
-        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
+    for i in 0..1 {
+        log::info!("sending message {}", i);
+        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
+            .await
+            .unwrap();
     }
 
-    thread::sleep(duration);
+    // wait for the message to be sent out
+    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
 
-    println!("stopping threads");
+    log::info!("fetching chats..");
+    let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
-    *running.write().unwrap() = false;
-    deltachat::job::interrupt_inbox_idle(&ctx);
-    deltachat::job::interrupt_smtp_idle(&ctx);
+    for i in 0..chats.len() {
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
+            .await
+            .unwrap();
+        log::info!("[{}] msg: {:?}", i, msg);
+    }
 
-    println!("joining");
-    t1.join().unwrap();
-    t2.join().unwrap();
-
-    println!("closing");
+    log::info!("stopping");
+    ctx.stop_io().await;
+    log::info!("closing");
+    drop(ctx);
+    events_spawn.await;
 }

python/README.rst

@@ -113,10 +113,10 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux1 based wheels
-================================
+Building manylinux based wheels
+====================================
 
-Building portable manylinux1 wheels which come with libdeltachat.so
+Building portable manylinux wheels which come with libdeltachat.so
 can be done with docker-tooling.
 
 using docker pull / premade images

python/examples/echo_and_quit.py

@@ -12,10 +12,13 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+            if message.is_system_message():
+                message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))
+            else:
+                text = message.text
+                message.chat.send_text("echoing from {}:\n{}".format(addr, text))
 
     @account_hookimpl
     def ac_message_delivered(self, message):

python/examples/group_tracking.py

@@ -12,24 +12,36 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))
 
     @account_hookimpl
-    def ac_configure_completed(self, success):
-        print("*** ac_configure_completed:", success)
+    def ac_outgoing_message(self, message):
+        print("ac_outgoing_message:", message)
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact):
-        print("*** ac_member_added", contact.addr, "from", chat)
+    def ac_configure_completed(self, success):
+        print("ac_configure_completed:", success)
+
+    @account_hookimpl
+    def ac_chat_modified(self, chat):
+        print("ac_chat_modified:", chat.id, chat.get_name())
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact):
-        print("*** ac_member_removed", contact.addr, "from", chat)
+    def ac_member_added(self, chat, contact, message):
+        print("ac_member_added {} to chat {} from {}".format(
+            contact.addr, chat.id, message.get_sender_contact().addr))
+        for member in chat.get_contacts():
+            print("chat member: {}".format(member.addr))
+
+    @account_hookimpl
+    def ac_member_removed(self, chat, contact, message):
+        print("ac_member_removed {} from chat {} by {}".format(
+            contact.addr, chat.id, message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -3,7 +3,7 @@ import pytest
 import py
 import echo_and_quit
 import group_tracking
-from deltachat.eventlogger import FFIEventLogger
+from deltachat.events import FFIEventLogger
 
 
 @pytest.fixture(scope='session')
@@ -17,41 +17,48 @@ def datadir():
         pytest.skip('test-data directory not found')
 
 
-def test_echo_quit_plugin(acfactory):
+def test_echo_quit_plugin(acfactory, lp):
+    lp.sec("creating one echo_and_quit bot")
     botproc = acfactory.run_bot_process(echo_and_quit)
 
+    lp.sec("creating a temp account to contact the bot")
     ac1 = acfactory.get_one_online_account()
+
+    lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
+    bot_chat = bot_contact.create_chat()
+    bot_chat.send_text("hello")
+
+    lp.sec("waiting for the reply message from the bot to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    assert reply.chat == ch1
-    ch1.send_text("/quit")
+    lp.sec("send quit sequence")
+    bot_chat.send_text("/quit")
     botproc.wait()
 
 
 def test_group_tracking_plugin(acfactory, lp):
     lp.sec("creating one group-tracking bot and two temp accounts")
-    botproc = acfactory.run_bot_process(group_tracking)
+    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
 
     ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
 
     botproc.fnmatch_lines("""
-        *ac_configure_completed: True*
+        *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+    ac1.add_account_plugin(FFIEventLogger(ac1))
+    ac2.add_account_plugin(FFIEventLogger(ac2))
 
-    lp.sec("creating bot test group with all three")
+    lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)
     ch = ac1.create_group_chat("bot test group")
     ch.add_contact(bot_contact)
     ch.send_text("hello")
 
     botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(ac1.get_config("addr")))
+        *ac_chat_modified*bot test group*
+    """)
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
     contact3 = ac1.create_contact(ac2.get_config("addr"))

python/fail_test.py

@@ -0,0 +1,7 @@
+from __future__ import print_function
+from deltachat import capi
+from deltachat.capi import ffi, lib
+
+if __name__ == "__main__":
+    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
+    lib.dc_stop_io(ctx)

python/install_python_bindings.py

@@ -19,6 +19,7 @@ if __name__ == "__main__":
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
     if target == 'release':
         cmd.append("--release")
+    print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -1,13 +1,13 @@
 import sys
 
-from . import capi, const, hookspec
-from .capi import ffi
+from . import capi, const, hookspec # noqa
+from .capi import ffi  # noqa
 from .account import Account  # noqa
 from .message import Message  # noqa
 from .contact import Contact  # noqa
 from .chat import Chat        # noqa
 from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import eventlogger
+from . import events
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -17,64 +17,6 @@ except DistributionNotFound:
     __version__ = "0.0.0.dev0-unknown"
 
 
-_DC_CALLBACK_MAP = {}
-
-
-@capi.ffi.def_extern()
-def py_dc_callback(ctx, evt, data1, data2):
-    """The global event handler.
-
-    CFFI only allows us to set one global event handler, so this one
-    looks up the correct event handler for the given context.
-    """
-    try:
-        callback = _DC_CALLBACK_MAP.get(ctx, lambda *a: 0)
-    except AttributeError:
-        # we are in a deep in GC-free/interpreter shutdown land
-        # nothing much better to do here than:
-        return 0
-
-    # the following code relates to the deltachat/_build.py's helper
-    # function which provides us signature info of an event call
-    evt_name = get_dc_event_name(evt)
-    event_sig_types = capi.lib.dc_get_event_signature_types(evt)
-    if data1 and event_sig_types & 1:
-        data1 = ffi.string(ffi.cast('char*', data1)).decode("utf8")
-    if data2 and event_sig_types & 2:
-        data2 = ffi.string(ffi.cast('char*', data2)).decode("utf8")
-        try:
-            if isinstance(data2, bytes):
-                data2 = data2.decode("utf8")
-        except UnicodeDecodeError:
-            # XXX ignoring the decode error is not quite correct but for now
-            # i don't want to hunt down encoding problems in the c lib
-            pass
-    try:
-        ret = callback(ctx, evt_name, data1, data2)
-        if ret is None:
-            ret = 0
-        assert isinstance(ret, int), repr(ret)
-        if event_sig_types & 4:
-            return ffi.cast('uintptr_t', ret)
-        elif event_sig_types & 8:
-            return ffi.cast('int', ret)
-    except:  # noqa
-        raise
-        ret = 0
-    return ret
-
-
-def set_context_callback(dc_context, func):
-    _DC_CALLBACK_MAP[dc_context] = func
-
-
-def clear_context_callback(dc_context):
-    try:
-        _DC_CALLBACK_MAP.pop(dc_context, None)
-    except AttributeError:
-        pass
-
-
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
         for name, val in vars(const).items():
@@ -97,7 +39,7 @@ def unregister_global_plugin(plugin):
     gm.unregister(plugin)
 
 
-register_global_plugin(eventlogger)
+register_global_plugin(events)
 
 
 def run_cmdline(argv=None, account_plugins=None):
@@ -118,9 +60,14 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = eventlogger.FFIEventLogger(ac, "bot")
+        ac.set_config("displayname", "bot")
+        log = events.FFIEventLogger(ac)
         ac.add_account_plugin(log)
 
+    for plugin in account_plugins or []:
+        print("adding plugin", plugin)
+        ac.add_account_plugin(plugin)
+
     if not ac.is_configured():
         assert args.email and args.password, (
             "you must specify --email and --password once to configure this database/account"
@@ -130,12 +77,11 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-
-    for plugin in account_plugins or []:
-        ac.add_account_plugin(plugin)
+        configtracker = ac.configure()
+        configtracker.wait_finish()
 
     # start IO threads and configure if neccessary
-    ac.start()
+    ac.start_io()
 
     print("{}: waiting for message".format(ac.get_config("addr")))
 

python/src/deltachat/_build.py

@@ -1,79 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            const char * dupstring_helper(const char* string)
-            {
-                return strdup(string);
-            }
-            int dc_get_event_signature_types(int e)
-            {
-                int result = 0;
-                if (DC_EVENT_DATA1_IS_STRING(e))
-                    result |= 1;
-                if (DC_EVENT_DATA2_IS_STRING(e))
-                    result |= 2;
-                if (DC_EVENT_RETURNS_STRING(e))
-                    result |= 4;
-                if (DC_EVENT_RETURNS_INT(e))
-                    result |= 8;
-                return result;
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern const char * dupstring_helper(const char* string);
-        extern int dc_get_event_signature_types(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -85,20 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
     builder.cdef("""
-        extern "Python" uintptr_t py_dc_callback(
-            dc_context_t* context,
-            int event,
-            uintptr_t data1,
-            uintptr_t data2);
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
     """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -1,22 +1,20 @@
 """ Account class implementation. """
 
 from __future__ import print_function
-import atexit
 from contextlib import contextmanager
 from email.utils import parseaddr
-import queue
 from threading import Event
 import os
 from array import array
-import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker
-from . import hookspec, iothreads
+from .tracker import ImexTracker, ConfigureTracker
+from . import hookspec
+from .events import EventThread
 
 
 class MissingCredentials(ValueError):
@@ -30,7 +28,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None):
+    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -39,42 +37,41 @@ class Account(object):
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
+        self._logging = logging
+
         self.add_account_plugin(self)
 
-        self._dc_context = ffi.gc(
-            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
-            _destroy_dc_context,
-        )
-
-        hook = hookspec.Global._get_plugin_manager().hook
-
-        self._threads = iothreads.IOThreads(self)
-        self._hook_event_queue = queue.Queue()
-        self._in_use_iter_events = False
-        self._shutdown_event = Event()
-
-        # open database
         self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
-        if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
-            raise ValueError("Could not dc_open: {}".format(db_path))
+
+        self._dc_context = ffi.gc(
+            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
+            lib.dc_context_unref,
+        )
+        if self._dc_context == ffi.NULL:
+            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
+
+        self._shutdown_event = Event()
+        self._event_thread = EventThread(self)
         self._configkeys = self.get_config("sys.config_keys").split()
-        atexit.register(self.shutdown)
+        hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    @hookspec.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        name, kwargs = self._map_ffi_event(ffi_event)
-        if name is not None:
-            ev = HookEvent(self, name=name, kwargs=kwargs)
-            self._hook_event_queue.put(ev)
+    def disable_logging(self):
+        """ disable logging. """
+        self._logging = False
+
+    def enable_logging(self):
+        """ re-enable logging. """
+        self._logging = True
 
     # def __del__(self):
     #    self.shutdown()
 
-    def ac_log_line(self, msg):
-        self._pm.hook.ac_log_line(message=msg)
+    def log(self, msg):
+        if self._logging:
+            self._pm.hook.ac_log_line(message=msg)
 
     def _check_config_key(self, name):
         if name not in self._configkeys:
@@ -163,7 +160,7 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return bool(lib.dc_is_configured(self._dc_context))
+        return True if lib.dc_is_configured(self._dc_context) else False
 
     def set_avatar(self, img_path):
         """Set self avatar.
@@ -216,22 +213,40 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, email, name=None):
-        """ create a (new) Contact. If there already is a Contact
-        with that e-mail address, it is unblocked and its name is
-        updated.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        :param email: email-address (text type)
-        :param name: display name for this contact (optional)
+        Calling this method will always resulut in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        `name` is updated if specified.
+
+        :param obj: email-address, Account or Contact instance.
+        :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
+        if isinstance(obj, Account):
+            if not obj.is_configured():
+                raise ValueError("can only add addresses from configured accounts")
+            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
+        elif isinstance(obj, Contact):
+            if obj.account != self:
+                raise ValueError("account mismatch {}".format(obj))
+            addr, displayname = obj.addr, obj.name
+        elif isinstance(obj, str):
+            displayname, addr = parseaddr(obj)
+        else:
+            raise TypeError("don't know how to create chat for %r" % (obj, ))
+
+        if name is None and displayname:
+            name = displayname
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -241,10 +256,25 @@ class Account(object):
         :returns: True if deletion succeeded (contact was deleted)
         """
         contact_id = contact.id
-        assert contact._dc_context == self._dc_context
+        assert contact.account == self
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
+    def get_contact_by_addr(self, email):
+        """ get a contact for the email address or None if it's blocked or doesn't exist. """
+        _, addr = parseaddr(email)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
+        if contact_id:
+            return self.get_contact_by_id(contact_id)
+
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -266,53 +296,37 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+    def get_fresh_messages(self):
+        """ yield all fresh messages from all chats. """
+        dc_array = ffi.gc(
+            lib.dc_get_fresh_msgs(self._dc_context),
+            lib.dc_array_unref
+        )
+        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(contact, "id"):
-            if contact._dc_context != self._dc_context:
-                raise ValueError("Contact belongs to a different Account")
-            contact_id = contact.id
-        else:
-            assert isinstance(contact, int)
-            contact_id = contact
-        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
-        return Chat(self, chat_id)
+    def create_chat(self, obj):
+        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        return self.create_contact(obj).create_chat()
 
-    def create_chat_by_message(self, message):
-        """ create or get an existing chat object for the
-        the specified message.
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
 
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
-        :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(message, "id"):
-            if self._dc_context != message._dc_context:
-                raise ValueError("Message belongs to a different Account")
-            msg_id = message.id
-        else:
-            assert isinstance(message, int)
-            msg_id = message
-        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
-        return Chat(self, chat_id)
-
-    def create_group_chat(self, name, verified=False):
+    def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
+        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        return Chat(self, chat_id)
+        chat = Chat(self, chat_id)
+        if contacts is not None:
+            for contact in contacts:
+                chat.add_contact(contact)
+        return chat
 
     def get_chats(self):
         """ return list of chats.
@@ -341,13 +355,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -413,8 +420,6 @@ class Account(object):
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -437,8 +442,6 @@ class Account(object):
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
             imex_tracker.wait_finish()
 
     def initiate_key_transfer(self):
@@ -447,8 +450,8 @@ class Account(object):
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
-        if not self._threads.is_started():
-            raise RuntimeError("threads not running, can not send out")
+        if not self.is_started():
+            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -530,6 +533,10 @@ class Account(object):
         self._pm.check_pending()
         return plugin
 
+    def remove_account_plugin(self, plugin, name=None):
+        """ remove an account plugin. """
+        self._pm.unregister(plugin, name=name)
+
     @contextmanager
     def temp_plugin(self, plugin):
         """ run a with-block with the given plugin temporarily registered. """
@@ -541,111 +548,77 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def start(self, callback_thread=True):
-        """ start this account (activate imap/smtp threads etc.)
-        and return immediately.
+    def start_io(self):
+        """ start this account's IO scheduling (Rust-core async scheduler)
 
-        If this account is not configured, an internal configuration
-        job will be scheduled if config values are sufficiently specified.
+        If this account is not configured an Exception is raised.
+        You need to call account.configure() and account.wait_configure_finish()
+        before.
 
-        You may call `wait_shutdown` or `shutdown` after the
-        account is in started mode.
+        You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
+        account is started.
 
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
+        :raises ConfigureFailed: if the account could not be configured.
 
         :returns: None
         """
         if not self.is_configured():
-            if not self.get_config("addr") or not self.get_config("mail_pw"):
-                raise MissingCredentials("addr or mail_pwd not set in config")
-            lib.dc_configure(self._dc_context)
-        self._threads.start(callback_thread=callback_thread)
+            raise ValueError("account not configured, cannot start io")
+        lib.dc_start_io(self._dc_context)
+
+    def configure(self):
+        """ Start configuration process and return a Configtracker instance
+        on which you can block with wait_finish() to get a True/False success
+        value for the configuration process.
+        """
+        assert not self.is_configured()
+        if not self.get_config("addr") or not self.get_config("mail_pw"):
+            raise MissingCredentials("addr or mail_pwd not set in config")
+        configtracker = ConfigureTracker(self)
+        self.add_account_plugin(configtracker)
+        lib.dc_configure(self._dc_context)
+        return configtracker
+
+    def is_started(self):
+        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
 
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def shutdown(self, wait=True):
-        """ shutdown account, stop threads and close and remove
-        underlying dc_context and callbacks. """
-        dc_context = self._dc_context
-        if dc_context is None:
+    def stop_io(self):
+        """ stop core IO scheduler if it is running. """
+        self.log("stop_ongoing")
+        self.stop_ongoing()
+
+        if bool(lib.dc_is_io_running(self._dc_context)):
+            self.log("dc_stop_io (stop core IO scheduler)")
+            lib.dc_stop_io(self._dc_context)
+        else:
+            self.log("stop_scheduler called on non-running context")
+
+    def shutdown(self):
+        """ shutdown and destroy account (stop callback thread, close and remove
+        underlying dc_context)."""
+        if self._dc_context is None:
             return
 
-        if self._threads.is_started():
-            self.stop_ongoing()
-            self._threads.stop(wait=False)
-        lib.dc_close(dc_context)
-        self._hook_event_queue.put(None)
-        self._threads.stop(wait=wait)  # to wait for threads
+        self.stop_io()
+
+        self.log("remove dc_context references")
+        # the dc_context_unref triggers get_next_event to return ffi.NULL
+        # which in turns makes the event thread finish execution
         self._dc_context = None
-        atexit.unregister(self.shutdown)
+
+        self.log("wait for event thread to finish")
+        self._event_thread.wait()
+
         self._shutdown_event.set()
+
         hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
-
-    def _handle_current_events(self):
-        """ handle all currently queued events and then return. """
-        while 1:
-            try:
-                event = self._hook_event_queue.get(block=False)
-            except queue.Empty:
-                break
-            else:
-                event.call_hook()
-
-    def iter_events(self, timeout=None):
-        """ yield hook events until shutdown.
-
-        It is not allowed to call iter_events() from multiple threads.
-        """
-        if self._in_use_iter_events:
-            raise RuntimeError("can only call iter_events() from one thread")
-        self._in_use_iter_events = True
-        while 1:
-            event = self._hook_event_queue.get(timeout=timeout)
-            if event is None:
-                break
-            yield event
-
-    def _map_ffi_event(self, ffi_event):
-        name = ffi_event.name
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-            data1 = ffi_event.data1
-            if data1 == 0 or data1 == 1000:
-                success = data1 == 1000
-                return "ac_configure_completed", dict(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = self.get_message_by_id(ffi_event.data2)
-            return "ac_incoming_message", dict(message=msg)
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = self.get_message_by_id(ffi_event.data2)
-                if msg.is_in_fresh():
-                    return "ac_incoming_message", dict(message=msg)
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = self.get_message_by_id(ffi_event.data2)
-            return "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_MEMBER_ADDED":
-            chat = self.get_chat_by_id(ffi_event.data1)
-            contact = self.get_contact_by_id(ffi_event.data2)
-            return "ac_member_added", dict(chat=chat, contact=contact)
-        elif name == "DC_EVENT_MEMBER_REMOVED":
-            chat = self.get_chat_by_id(ffi_event.data1)
-            contact = self.get_contact_by_id(ffi_event.data2)
-            return "ac_member_removed", dict(chat=chat, contact=contact)
-        return None, {}
-
-
-def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
-    # destructor for dc_context
-    dc_context_unref(dc_context)
-    try:
-        deltachat.clear_context_callback(dc_context)
-    except (TypeError, AttributeError):
-        # we are deep into Python Interpreter shutdown,
-        # so no need to clear the callback context mapping.
-        pass
+        hook.dc_account_after_shutdown(account=self)
+        self.log("shutdown finished")
 
 
 class ScannedQRCode:
@@ -661,17 +634,3 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
-
-
-class HookEvent:
-    def __init__(self, account, name, kwargs):
-        assert hasattr(account._pm.hook, name), name
-        self.account = account
-        self.name = name
-        self.kwargs = kwargs
-
-    def call_hook(self):
-        hook = getattr(self.account._pm.hook, self.name, None)
-        if hook is None:
-            raise ValueError("event_name {} unknown".format(self.name))
-        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -18,24 +18,25 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
-        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
-               self._dc_context == getattr(other, "_dc_context", None)
+               self.account._dc_context == other.account._dc_context
 
     def __ne__(self, other):
         return not (self == other)
 
     def __repr__(self):
-        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
+        return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
     def _dc_chat(self):
         return ffi.gc(
-            lib.dc_get_chat(self._dc_context, self.id),
+            lib.dc_get_chat(self.account._dc_context, self.id),
             lib.dc_chat_unref
         )
 
@@ -47,7 +48,7 @@ class Chat(object):
         - does not delete messages on server
         - the chat or contact is not blocked, new message will arrive
         """
-        lib.dc_delete_chat(self._dc_context, self.id)
+        lib.dc_delete_chat(self.account._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
@@ -105,7 +106,7 @@ class Chat(object):
         :returns: None
         """
         name = as_dc_charpointer(name)
-        return lib.dc_set_chat_name(self._dc_context, self.id, name)
+        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
 
     def mute(self, duration=None):
         """ mutes the chat
@@ -117,7 +118,7 @@ class Chat(object):
             mute_duration = -1
         else:
             mute_duration = duration
-        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, mute_duration)
+        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration)
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
@@ -126,7 +127,7 @@ class Chat(object):
 
         :returns: None
         """
-        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, 0)
+        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, 0)
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
@@ -152,7 +153,7 @@ class Chat(object):
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
         where account.join_with_qrcode(qr) needs to be called.
         """
-        res = lib.dc_get_securejoin_qr(self._dc_context, self.id)
+        res = lib.dc_get_securejoin_qr(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
     # ------  chat messaging API ------------------------------
@@ -174,7 +175,7 @@ class Chat(object):
             assert msg.id != 0
             # get a fresh copy of dc_msg, the core needs it
             msg = Message.from_db(self.account, msg.id)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         # modify message in place to avoid bad state for the caller
@@ -189,7 +190,7 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         msg = as_dc_charpointer(text)
-        msg_id = lib.dc_send_text_msg(self._dc_context, self.id, msg)
+        msg_id = lib.dc_send_text_msg(self.account._dc_context, self.id, msg)
         if msg_id == 0:
             raise ValueError("message could not be send, does chat exist?")
         return Message.from_db(self.account, msg_id)
@@ -204,7 +205,7 @@ class Chat(object):
         """
         msg = Message.new_empty(self.account, view_type="file")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
@@ -219,7 +220,7 @@ class Chat(object):
         mime_type = mimetypes.guess_type(path)[0]
         msg = Message.new_empty(self.account, view_type="image")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
@@ -230,7 +231,7 @@ class Chat(object):
         :param msg: the message to be prepared.
         :returns: :class:`deltachat.message.Message` instance.
         """
-        msg_id = lib.dc_prepare_msg(self._dc_context, self.id, msg._dc_msg)
+        msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
         # invalidate passed in message which is not safe to use anymore
@@ -266,7 +267,7 @@ class Chat(object):
         msg = Message.from_db(self.account, message.id)
 
         # pass 0 as chat-id because core-docs say it's ok when out-preparing
-        sent_id = lib.dc_send_msg(self._dc_context, 0, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, 0, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         assert sent_id == msg.id
@@ -280,9 +281,9 @@ class Chat(object):
         :returns: None
         """
         if message is None:
-            lib.dc_set_draft(self._dc_context, self.id, ffi.NULL)
+            lib.dc_set_draft(self.account._dc_context, self.id, ffi.NULL)
         else:
-            lib.dc_set_draft(self._dc_context, self.id, message._dc_msg)
+            lib.dc_set_draft(self.account._dc_context, self.id, message._dc_msg)
 
     def get_draft(self):
         """ get draft message for this chat.
@@ -290,7 +291,7 @@ class Chat(object):
         :param message: a :class:`Message` instance
         :returns: Message object or None (if no draft available)
         """
-        x = lib.dc_get_draft(self._dc_context, self.id)
+        x = lib.dc_get_draft(self.account._dc_context, self.id)
         if x == ffi.NULL:
             return None
         dc_msg = ffi.gc(x, lib.dc_msg_unref)
@@ -302,7 +303,7 @@ class Chat(object):
         :returns: list of :class:`deltachat.message.Message` objects for this chat.
         """
         dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(self._dc_context, self.id, 0, 0),
+            lib.dc_get_chat_msgs(self.account._dc_context, self.id, 0, 0),
             lib.dc_array_unref
         )
         return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
@@ -312,60 +313,69 @@ class Chat(object):
 
         :returns: number of fresh messages
         """
-        return lib.dc_get_fresh_msg_cnt(self._dc_context, self.id)
+        return lib.dc_get_fresh_msg_cnt(self.account._dc_context, self.id)
 
     def mark_noticed(self):
         """ mark all messages in this chat as noticed.
 
         Noticed messages are no longer fresh.
         """
-        return lib.dc_marknoticed_chat(self._dc_context, self.id)
+        return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
 
     def get_summary(self):
         """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
+        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
         s = from_dc_charpointer(dc_res)
         return json.loads(s)
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, contact):
+    def add_contact(self, obj):
         """ add a contact to this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
         :returns: None
         """
-        ret = lib.dc_add_contact_to_chat(self._dc_context, self.id, contact.id)
+        contact = self.account.create_contact(obj)
+        ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
+        return contact
 
-    def remove_contact(self, contact):
+    def remove_contact(self, obj):
         """ remove a contact from this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        ret = lib.dc_remove_contact_from_chat(self._dc_context, self.id, contact.id)
+        contact = self.account.create_contact(obj)
+        ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
-        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-
         """
         from .contact import Contact
         dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self._dc_context, self.id),
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
             lib.dc_array_unref
         )
         return list(iter_array(
             dc_array, lambda id: Contact(self.account, id))
         )
 
+    def num_contacts(self):
+        """ return number of contacts in this chat. """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return lib.dc_array_get_cnt(dc_array)
+
     def set_profile_image(self, img_path):
         """Set group profile image.
 
@@ -378,7 +388,7 @@ class Chat(object):
         """
         assert os.path.exists(img_path), img_path
         p = as_dc_charpointer(img_path)
-        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, p)
+        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, p)
         if res != 1:
             raise ValueError("Setting Profile Image {!r} failed".format(p))
 
@@ -391,7 +401,7 @@ class Chat(object):
         :raises ValueError: if profile image could not be reset
         :returns: None
         """
-        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, ffi.NULL)
+        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, ffi.NULL)
         if res != 1:
             raise ValueError("Removing Profile Image failed")
 
@@ -415,19 +425,13 @@ class Chat(object):
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
-    def get_subtitle(self):
-        """return the subtitle of the chat
-        :returns: the subtitle
-        """
-        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
-
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
         """return True if this chat has location-sending enabled currently.
         :returns: True if location sending is enabled.
         """
-        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
+        return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
 
     def is_archived(self):
         """return True if this chat is archived.
@@ -440,7 +444,7 @@ class Chat(object):
 
         all subsequent messages will carry a location with them.
         """
-        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
+        lib.dc_send_locations_to_chat(self.account._dc_context, self.id, seconds)
 
     def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
         """return list of locations for the given contact in the given timespan.
@@ -464,7 +468,7 @@ class Chat(object):
         else:
             contact_id = contact.id
 
-        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
+        dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to)
         return [
             Location(
                 latitude=lib.dc_array_get_latitude(dc_array, i),

python/src/deltachat/const.py

@@ -1,202 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
-DC_EVENT_MEMBER_ADDED = 2063
-DC_EVENT_MEMBER_REMOVED = 2064
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_MEMBER = 4
-DC_STR_CONTACT = 6
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_SELFTALK_SUBTITLE = 50
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -3,6 +3,8 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
+from .chat import Chat
+from . import const
 
 
 class Contact(object):
@@ -11,23 +13,24 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
-        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
-        return self._dc_context == other._dc_context and self.id == other.id
+        return self.account._dc_context == other.account._dc_context and self.id == other.id
 
     def __ne__(self, other):
         return not (self == other)
 
     def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self.account._dc_context)
 
     @property
     def _dc_contact(self):
         return ffi.gc(
-            lib.dc_get_contact(self._dc_context, self.id),
+            lib.dc_get_contact(self.account._dc_context, self.id),
             lib.dc_contact_unref
         )
 
@@ -37,10 +40,13 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def display_name(self):
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
+    # deprecated alias
+    display_name = name
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -59,6 +65,16 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)
+    def create_chat(self):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        dc_context = self.account._dc_context
+        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
+        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
+        return Chat(self.account, chat_id)
+
+    # deprecated name
+    get_chat = create_chat

python/src/deltachat/direct_imap.py

@@ -0,0 +1,213 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
+import io
+import email
+import ssl
+import pathlib
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
+
+
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+ALL = "1:*"
+
+
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account):
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        ssl_context = ssl.create_default_context()
+
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
+        try:
+            self.conn.logout()
+        except (OSError, IMAPClientError):
+            print("Could not logout direct_imap conn")
+
+    def select_folder(self, foldername):
+        assert not self._idling
+        return self.conn.select_folder(foldername)
+
+    def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
+        if "_" not in config_name:
+            config_name = "configured_{}_folder".format(config_name)
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+
+    def list_folders(self):
+        """ return list of all existing folder names"""
+        assert not self._idling
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
+
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self):
+        assert not self._idling
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
+        assert not self._idling
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.set_flags(messages, [SEEN])
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self):
+        return len(self.get_unread_messages())
+
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[HEADER]']
+                flags = data[FLAGS]
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/eventlogger.py

@@ -1,137 +0,0 @@
-import deltachat
-import threading
-import time
-import re
-from queue import Queue, Empty
-from .hookspec import account_hookimpl, global_hookimpl
-
-
-@global_hookimpl
-def dc_account_init(account):
-    # send all FFI events for this account to a plugin hook
-    def _ll_event(ctx, evt_name, data1, data2):
-        assert ctx == account._dc_context
-        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-        account._pm.hook.ac_process_ffi_event(
-            account=account, ffi_event=ffi_event
-        )
-    deltachat.set_context_callback(account._dc_context, _ll_event)
-
-
-@global_hookimpl
-def dc_account_after_shutdown(dc_context):
-    deltachat.clear_context_callback(dc_context)
-
-
-class FFIEvent:
-    def __init__(self, name, data1, data2):
-        self.name = name
-        self.data1 = data1
-        self.data2 = data2
-
-    def __str__(self):
-        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
-
-
-class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
-    you'll get all ffi-events printed.
-    """
-    # to prevent garbled logging
-    _loglock = threading.RLock()
-
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
-        self.account = account
-        self.logid = logid
-        self.init_time = time.time()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._log_event(ffi_event)
-
-    def _log_event(self, ffi_event):
-        # don't show events that are anyway empty impls now
-        if ffi_event.name == "DC_EVENT_GET_STRING":
-            return
-        self.account.ac_log_line(str(ffi_event))
-
-    @account_hookimpl
-    def ac_log_line(self, message):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        elapsed = time.time() - self.init_time
-        locname = tname
-        if self.logid:
-            locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-        with self._loglock:
-            print(s, flush=True)
-
-
-class FFIEventTracker:
-    def __init__(self, account, timeout=None):
-        self.account = account
-        self._timeout = timeout
-        self._event_queue = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._event_queue.put(ffi_event)
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get(check_error=check_error)
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout if timeout is not None else self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError(str(ev))
-        return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev.name):
-                return ev
-
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
-                return ev
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.account.get_message_by_id(ev.data2)
-
-    def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
-        if the event contains no msgid"""
-        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-        if ev.data2 > 0:
-            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/events.py

@@ -0,0 +1,219 @@
+import threading
+import time
+import re
+from queue import Queue, Empty
+
+import deltachat
+from .hookspec import account_hookimpl
+from contextlib import contextmanager
+from .capi import ffi, lib
+from .message import map_system_message
+from .cutil import from_dc_charpointer
+
+
+class FFIEvent:
+    def __init__(self, name, data1, data2):
+        self.name = name
+        self.data1 = data1
+        self.data2 = data2
+
+    def __str__(self):
+        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
+
+
+class FFIEventLogger:
+    """ If you register an instance of this logger with an Account
+    you'll get all ffi-events printed.
+    """
+    # to prevent garbled logging
+    _loglock = threading.RLock()
+
+    def __init__(self, account):
+        self.account = account
+        self.logid = self.account.get_config("displayname")
+        self.init_time = time.time()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self.account.log(str(ffi_event))
+
+    @account_hookimpl
+    def ac_log_line(self, message):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        elapsed = time.time() - self.init_time
+        locname = tname
+        if self.logid:
+            locname += "-" + self.logid
+        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        with self._loglock:
+            print(s, flush=True)
+
+
+class FFIEventTracker:
+    def __init__(self, account, timeout=None):
+        self.account = account
+        self._timeout = timeout
+        self._event_queue = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._event_queue.put(ffi_event)
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get(check_error=check_error)
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout if timeout is not None else self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev.name == "DC_EVENT_ERROR":
+            raise ValueError("unexpected event: {}".format(ev))
+        return ev
+
+    def iter_events(self, timeout=None, check_error=True):
+        while 1:
+            yield self.get(timeout=timeout, check_error=check_error)
+
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        for ev in self.iter_events(timeout=timeout, check_error=check_error):
+            if rex.match(ev.name):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev.data2):
+                return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev.name), "event found {}".format(ev)
+
+    def wait_securejoin_inviter_progress(self, target):
+        while 1:
+            event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+            if event.data2 >= target:
+                print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
+                break
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.account.get_message_by_id(ev.data2)
+
+    def wait_next_messages_changed(self):
+        """ wait for and return next message-changed message or None
+        if the event contains no msgid"""
+        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
+        if ev.data2 > 0:
+            return self.account.get_message_by_id(ev.data2)
+
+    def wait_msg_delivered(self, msg):
+        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == msg.chat.id
+        assert ev.data2 == msg.id
+        assert msg.is_out_delivered()
+
+
+class EventThread(threading.Thread):
+    """ Event Thread for an account.
+
+    With each Account init this callback thread is started.
+    """
+    def __init__(self, account):
+        self.account = account
+        super(EventThread, self).__init__(name="events")
+        self.setDaemon(True)
+        self.start()
+
+    @contextmanager
+    def log_execution(self, message):
+        self.account.log(message + " START")
+        yield
+        self.account.log(message + " FINISHED")
+
+    def wait(self):
+        if self == threading.current_thread():
+            # we are in the callback thread and thus cannot
+            # wait for the thread-loop to finish.
+            return
+        self.join()
+
+    def run(self):
+        """ get and run events until shutdown. """
+        with self.log_execution("EVENT THREAD"):
+            self._inner_run()
+
+    def _inner_run(self):
+        event_emitter = ffi.gc(
+            lib.dc_get_event_emitter(self.account._dc_context),
+            lib.dc_event_emitter_unref,
+        )
+        while 1:
+            event = lib.dc_get_next_event(event_emitter)
+            if event == ffi.NULL:
+                break
+            evt = lib.dc_event_get_id(event)
+            data1 = lib.dc_event_get_data1_int(event)
+            # the following code relates to the deltachat/_build.py's helper
+            # function which provides us signature info of an event call
+            evt_name = deltachat.get_dc_event_name(evt)
+            if lib.dc_event_has_string_data(evt):
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
+            else:
+                data2 = lib.dc_event_get_data2_int(event)
+
+            lib.dc_event_unref(event)
+            ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+            try:
+                self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
+                for name, kwargs in self._map_ffi_event(ffi_event):
+                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
+                    hook = getattr(self.account._pm.hook, name)
+                    hook(**kwargs)
+            except Exception:
+                if self.account._dc_context is not None:
+                    raise
+
+    def _map_ffi_event(self, ffi_event):
+        name = ffi_event.name
+        account = self.account
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+            data1 = ffi_event.data1
+            if data1 == 0 or data1 == 1000:
+                success = data1 == 1000
+                yield "ac_configure_completed", dict(success=success)
+        elif name == "DC_EVENT_INCOMING_MSG":
+            msg = account.get_message_by_id(ffi_event.data2)
+            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+        elif name == "DC_EVENT_MSGS_CHANGED":
+            if ffi_event.data2 != 0:
+                msg = account.get_message_by_id(ffi_event.data2)
+                if msg.is_outgoing():
+                    res = map_system_message(msg)
+                    if res and res[0].startswith("ac_member"):
+                        yield res
+                    yield "ac_outgoing_message", dict(message=msg)
+                elif msg.is_in_fresh():
+                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+        elif name == "DC_EVENT_MSG_DELIVERED":
+            msg = account.get_message_by_id(ffi_event.data2)
+            yield "ac_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_CHAT_MODIFIED":
+            chat = account.get_chat_by_id(ffi_event.data1)
+            yield "ac_chat_modified", dict(chat=chat)

python/src/deltachat/hookspec.py

@@ -15,8 +15,9 @@ global_hookimpl = pluggy.HookimplMarker(global_spec_name)
 class PerAccount:
     """ per-Account-instance hook specifications.
 
-    Except for ac_process_ffi_event all hooks are executed
-    in the thread which calls Account.wait_shutdown().
+    All hooks are executed in a dedicated Event thread.
+    Hooks are not allowed to block/last long as this
+    blocks overall event processing on the python side.
     """
     @classmethod
     def _make_plugin_manager(cls):
@@ -42,23 +43,31 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
         """ Called on any incoming message (to deaddrop or chat). """
 
+    @account_hookspec
+    def ac_outgoing_message(self, message):
+        """ Called on each outgoing message (both system and "normal")."""
+
     @account_hookspec
     def ac_message_delivered(self, message):
         """ Called when an outgoing message has been delivered to SMTP. """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact):
-        """ Called for each contact added to a chat. """
+    def ac_chat_modified(self, chat):
+        """ Chat was created or modified regarding membership, avatar, title. """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact):
-        """ Called for each contact removed from a chat. """
+    def ac_member_added(self, chat, contact, message):
+        """ Called for each contact added to an accepted chat. """
+
+    @account_hookspec
+    def ac_member_removed(self, chat, contact, message):
+        """ Called for each contact removed from a chat.  """
 
 
 class Global:
@@ -80,5 +89,13 @@ class Global:
         """ called when `Account::__init__()` function starts executing. """
 
     @global_hookspec
-    def dc_account_after_shutdown(self, account, dc_context):
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
+    @global_hookspec
+    def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -1,106 +0,0 @@
-
-import threading
-import time
-
-from contextlib import contextmanager
-
-from .capi import lib
-
-
-class IOThreads:
-    def __init__(self, account):
-        self.account = account
-        self._dc_context = account._dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-
-    def is_started(self):
-        return len(self._name2thread) > 0
-
-    def start(self, callback_thread):
-        assert not self.is_started()
-        self._start_one_thread("inbox", self.imap_thread_run)
-        self._start_one_thread("smtp", self.smtp_thread_run)
-
-        if callback_thread:
-            self._start_one_thread("cb", self.cb_thread_run)
-
-        if int(self.account.get_config("mvbox_watch")):
-            self._start_one_thread("mvbox", self.mvbox_thread_run)
-
-        if int(self.account.get_config("sentbox_watch")):
-            self._start_one_thread("sentbox", self.sentbox_thread_run)
-
-    def _start_one_thread(self, name, func):
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        t.setDaemon(1)
-        t.start()
-
-    @contextmanager
-    def log_execution(self, message):
-        self.account.ac_log_line(message + " START")
-        yield
-        self.account.ac_log_line(message + " FINISHED")
-
-    def stop(self, wait=False):
-        self._thread_quitflag = True
-
-        # Workaround for a race condition. Make sure that thread is
-        # not in between checking for quitflag and entering idle.
-        time.sleep(0.5)
-
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        if "mvbox" in self._name2thread:
-            lib.dc_interrupt_mvbox_idle(self._dc_context)
-        if "sentbox" in self._name2thread:
-            lib.dc_interrupt_sentbox_idle(self._dc_context)
-        if wait:
-            for name, thread in self._name2thread.items():
-                if thread != threading.currentThread():
-                    thread.join()
-
-    def cb_thread_run(self):
-        with self.log_execution("CALLBACK THREAD START"):
-            it = self.account.iter_events()
-            while not self._thread_quitflag:
-                try:
-                    ev = next(it)
-                except StopIteration:
-                    break
-                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
-                ev.call_hook()
-
-    def imap_thread_run(self):
-        with self.log_execution("INBOX THREAD START"):
-            while not self._thread_quitflag:
-                lib.dc_perform_imap_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_idle(self._dc_context)
-
-    def mvbox_thread_run(self):
-        with self.log_execution("MVBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_mvbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_idle(self._dc_context)
-
-    def sentbox_thread_run(self):
-        with self.log_execution("SENTBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_sentbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_idle(self._dc_context)
-
-    def smtp_thread_run(self):
-        with self.log_execution("SMTP THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_smtp_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -16,8 +16,7 @@ class Message(object):
     """
     def __init__(self, account, dc_msg):
         self.account = account
-        self._dc_context = account._dc_context
-        assert isinstance(self._dc_context, ffi.CData)
+        assert isinstance(self.account._dc_context, ffi.CData)
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
@@ -28,7 +27,11 @@ class Message(object):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
+        c = self.get_sender_contact()
+        typ = "outgoing" if self.is_outgoing() else "incoming"
+        return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
+            typ, self.is_system_message(), repr(self.text[:10]),
+            self.id, c.id, c.addr, self.chat.id, self.chat.get_name())
 
     @classmethod
     def from_db(cls, account, id):
@@ -50,15 +53,19 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
+    def create_chat(self):
+        """ create or get an existing chat (group) object for this message.
+
+        If the message is a deaddrop contact request
+        the sender will become an accepted contact.
+
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self._dc_context, self.id),
-                lib.dc_msg_unref
-        )
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):
@@ -91,6 +98,10 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
+    def is_system_message(self):
+        """ return True if this message is a system/info message. """
+        return bool(lib.dc_msg_is_info(self._dc_msg))
+
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
@@ -112,12 +123,12 @@ class Message(object):
 
         The text is multiline and may contain eg. the raw text of the message.
         """
-        return from_dc_charpointer(lib.dc_get_msg_info(self._dc_context, self.id))
+        return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
 
     def continue_key_transfer(self, setup_code):
         """ extract key and use it as primary key for this account. """
         res = lib.dc_continue_key_transfer(
-                self._dc_context,
+                self.account._dc_context,
                 self.id,
                 as_dc_charpointer(setup_code)
         )
@@ -152,7 +163,7 @@ class Message(object):
         :returns: email-mime message object (with headers only, no body).
         """
         import email.parser
-        mime_headers = lib.dc_get_mime_headers(self._dc_context, self.id)
+        mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
             if isinstance(s, bytes):
@@ -195,7 +206,7 @@ class Message(object):
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
-                lib.dc_get_msg(self._dc_context, self.id),
+                lib.dc_get_msg(self.account._dc_context, self.id),
                 lib.dc_msg_unref
             )
         return lib.dc_msg_get_state(dc_msg)
@@ -224,6 +235,13 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
+    def is_outgoing(self):
+        """Return True if Message is outgoing. """
+        return self._msgstate in (
+            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
+            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
+            const.DC_STATE_OUT_DELIVERED)
+
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -309,3 +327,29 @@ def get_viewtype_code_from_name(view_type_name):
             return code
     raise ValueError("message typecode not found for {!r}, "
                      "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
+
+
+#
+# some helper code for turning system messages into hook events
+#
+
+def map_system_message(msg):
+    if msg.is_system_message():
+        res = parse_system_add_remove(msg.text)
+        if res:
+            contact = msg.account.get_contact_by_addr(res[1])
+            if contact:
+                d = dict(chat=msg.chat, contact=contact, message=msg)
+                return "ac_member_" + res[0], d
+
+
+def parse_system_add_remove(text):
+    # Member Me (x@y) removed by a@b.
+    # Member x@y removed by a@b
+    text = text.lower()
+    parts = text.split()
+    if parts[0] == "member":
+        if parts[2] in ("removed", "added"):
+            return parts[2], parts[1]
+        if parts[3] in ("removed", "added"):
+            return parts[3], parts[2].strip("()")

python/src/deltachat/testplugin.py

@@ -1,21 +1,25 @@
 from __future__ import print_function
 import os
 import sys
+import io
 import subprocess
 import queue
 import threading
 import fnmatch
+import time
+import weakref
+import tempfile
+
 import pytest
 import requests
-import time
-from . import Account, const
-from .tracker import ConfigureTracker
-from .capi import lib
-from .eventlogger import FFIEventLogger, FFIEventTracker
-from _pytest.monkeypatch import MonkeyPatch
-from _pytest._code import Source
 
-import tempfile
+from . import Account, const
+from .capi import lib
+from .events import FFIEventLogger, FFIEventTracker
+from _pytest._code import Source
+from deltachat import direct_imap
+
+import deltachat
 
 
 def pytest_addoption(parser):
@@ -31,38 +35,80 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
         if cfg:
             config.option.liveconfig = cfg
 
+    # Make sure we don't get garbled output because threads keep running
+    # collect all ever created accounts in a weakref-set (so we don't
+    # keep objects unneccessarily alive) and enable/disable logging
+    # for each pytest test phase # (setup/call/teardown).
+    # Additionally make the acfactory use a logging/no-logging default.
 
-def pytest_runtest_setup(item):
-    if (list(item.iter_markers(name="ignored"))
-            and not item.config.getoption("ignored")):
-        pytest.skip("Ignored tests not requested, use --ignored")
+    class LoggingAspect:
+        def __init__(self):
+            self._accounts = weakref.WeakSet()
+
+        @deltachat.global_hookimpl
+        def dc_account_init(self, account):
+            self._accounts.add(account)
+
+        def disable_logging(self, item):
+            for acc in self._accounts:
+                acc.disable_logging()
+            acfactory = item.funcargs.get("acfactory")
+            if acfactory:
+                acfactory.set_logging_default(False)
+
+        def enable_logging(self, item):
+            for acc in self._accounts:
+                acc.enable_logging()
+            acfactory = item.funcargs.get("acfactory")
+            if acfactory:
+                acfactory.set_logging_default(True)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_runtest_setup(self, item):
+            if item.get_closest_marker("ignored"):
+                if not item.config.getvalue("ignored"):
+                    pytest.skip("use --ignored to run this test")
+            self.enable_logging(item)
+            yield
+            self.disable_logging(item)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_pyfunc_call(self, pyfuncitem):
+            self.enable_logging(pyfuncitem)
+            yield
+            self.disable_logging(pyfuncitem)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_runtest_teardown(self, item):
+            self.enable_logging(item)
+            yield
+            self.disable_logging(item)
+
+    la = LoggingAspect()
+    config.pluginmanager.register(la)
+    deltachat.register_global_plugin(la)
 
 
 def pytest_report_header(config, startdir):
     summary = []
 
     t = tempfile.mktemp()
-    m = MonkeyPatch()
     try:
-        m.setattr(sys.stdout, "write", lambda x: len(x))
         ac = Account(t)
         info = ac.get_info()
         ac.shutdown()
     finally:
-        m.undo()
         os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
+    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
          info['deltachat_core_version'],
          info['sqlite_version'],
+         info['journal_mode'],
      )])
 
     cfg = config.option.liveconfig
@@ -164,24 +210,37 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
+            self._accounts = []
             self.init_time = time.time()
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
+            self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
                 fin = self._finalizers.pop()
                 fin()
 
+            while self._accounts:
+                acc = self._accounts.pop()
+                acc.shutdown()
+                acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
+
         def make_account(self, path, logid, quiet=False):
-            ac = Account(path)
+            ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
+            ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
-            self._finalizers.append(ac.shutdown)
+                ac.add_account_plugin(FFIEventLogger(ac))
+            self._accounts.append(ac)
             return ac
 
+        def set_logging_default(self, logging):
+            self._logging = bool(logging)
+
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
@@ -244,24 +303,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac.start()
+            ac._configtracker = ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1._configtracker.wait_imap_connected()
-            ac1._configtracker.wait_smtp_connected()
-            ac1._configtracker.wait_finish()
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1._configtracker.wait_finish()
-            ac2._configtracker.wait_finish()
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+                        for i in range(num)]
+            self.wait_configure_and_start_io()
+            for acc in accounts:
+                acc.add_account_plugin(FFIEventLogger(acc))
+            return accounts
+
         def clone_online_account(self, account, pre_generated_key=True):
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
@@ -277,29 +341,46 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac.start()
+            ac._configtracker = ac.configure()
             return ac
 
-        def run_bot_process(self, module):
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
+
+        def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
+                "-u",
                 fn,
-                "--show-ffi",
                 "--email", bot_cfg["addr"],
                 "--password", bot_cfg["mail_pw"],
                 bot_ac.db_path,
             ]
+            if ffi:
+                args.insert(-1, "--show-ffi")
             print("$", " ".join(args))
             popen = subprocess.Popen(
                 args=args,
                 stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=1,                 # line buffering
+                bufsize=0,                 # line buffering
                 close_fds=True,            # close all FDs other than 0/1/2
                 universal_newlines=True    # give back text
             )
@@ -307,9 +388,42 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    return am
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
@@ -332,6 +446,7 @@ class BotProcess:
                     break
                 line = line.strip()
                 self.stdout_queue.put(line)
+                print("bot-stdout: ", line)
         finally:
             self.stdout_queue.put(None)
 
@@ -345,15 +460,21 @@ class BotProcess:
         patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
         for next_pattern in patterns:
             print("+++FNMATCH:", next_pattern)
+            ignored = []
             while 1:
                 line = self.stdout_queue.get(timeout=15)
                 if line is None:
+                    if ignored:
+                        print("BOT stdout terminated after these lines")
+                        for line in ignored:
+                            print(line)
                     raise IOError("BOT stdout-thread terminated")
                 if fnmatch.fnmatch(line, next_pattern):
                     print("+++MATCHED:", line)
                     break
                 else:
                     print("+++IGN:", line)
+                    ignored.append(line)
 
 
 @pytest.fixture
@@ -370,4 +491,17 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
+
     return Printer()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # set a report attribute for each phase of a call, which can
+    # be "setup", "call", "teardown"
+
+    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -18,7 +18,7 @@ class ImexTracker:
         if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
             self._imex_events.put(ffi_event.data1)
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(ffi_event.data1)
+            self._imex_events.put(ffi_event.data2)
 
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
@@ -40,11 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
+        self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -53,10 +56,15 @@ class ConfigureTracker:
             self._smtp_finished.set()
         elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
             self._imap_finished.set()
+        elif ffi_event.name == "DC_EVENT_CONFIGURE_PROGRESS":
+            self._progress.put(ffi_event.data1)
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
+        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """
@@ -66,6 +74,12 @@ class ConfigureTracker:
         """ wait until smtp is configured. """
         self._imap_finished.wait()
 
+    def wait_progress(self, data1=None):
+        while 1:
+            evdata = self._progress.get()
+            if data1 is None or evdata == data1:
+                break
+
     def wait_finish(self):
         """ wait until configure is completed.
 

python/tests/auditwheels.py

@@ -10,4 +10,6 @@ if __name__ == "__main__":
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
-            subprocess.check_call(["auditwheel", "repair", p, "-w", workspacedir])
+            subprocess.check_call(
+                ["auditwheel", "repair", p, "-w", workspacedir,
+                 "--plat", "manylinux2014_x86_64"])

python/tests/conftest.py

@@ -1,18 +0,0 @@
-from __future__ import print_function
-
-
-def wait_configuration_progress(account, min_target, max_target=1001):
-    min_target = min(min_target, max_target)
-    while 1:
-        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        if event.data1 >= min_target and event.data1 <= max_target:
-            print("** CONFIG PROGRESS {}".format(min_target), account)
-            break
-
-
-def wait_securejoin_inviter_progress(account, target):
-    while 1:
-        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-        if event.data2 >= target:
-            print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
-            break

python/tests/stress_test_db.py

@@ -0,0 +1,135 @@
+
+import time
+import threading
+import pytest
+import os
+from queue import Queue, Empty
+
+import deltachat
+
+
+def test_db_busy_error(acfactory, tmpdir):
+    starttime = time.time()
+    log_lock = threading.RLock()
+
+    def log(string):
+        with log_lock:
+            print("%3.2f %s" % (time.time() - starttime, string))
+
+    # make a number of accounts
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
+    log("created %s accounts" % len(accounts))
+
+    # put a bigfile into each account
+    for acc in accounts:
+        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
+        with open(acc.bigfile, "wb") as f:
+            f.write(b"01234567890"*1000_000)
+    log("created %s bigfiles" % len(accounts))
+
+    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
+    chat = accounts[0].create_group_chat("stress-group")
+    for addr in contact_addrs[1:]:
+        chat.add_contact(chat.account.create_contact(addr))
+
+    # setup auto-responder bots which report back failures/actions
+    report_queue = Queue()
+
+    def report_func(replier, report_type, *report_args):
+        report_queue.put((replier, report_type, report_args))
+
+    # each replier receives all events and sends report events to receive_queue
+    repliers = []
+    for acc in accounts:
+        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
+        acc.add_account_plugin(replier)
+        repliers.append(replier)
+
+    # kick off message sending
+    # after which repliers will reply to each other
+    chat.send_text("hello")
+
+    alive_count = len(accounts)
+    while alive_count > 0:
+        try:
+            replier, report_type, report_args = report_queue.get(timeout=10)
+        except Empty:
+            log("timeout waiting for next event")
+            pytest.fail("timeout exceeded")
+        if report_type == ReportType.exit:
+            replier.log("EXIT")
+        elif report_type == ReportType.ffi_error:
+            replier.log("ERROR: {}".format(report_args[0]))
+        elif report_type == ReportType.message_echo:
+            continue
+        else:
+            raise ValueError("{} unknown report type {}, args={}".format(
+                addr, report_type, report_args
+            ))
+        alive_count -= 1
+        replier.log("shutting down")
+        replier.account.shutdown()
+        replier.log("shut down complete, remaining={}".format(alive_count))
+
+
+class ReportType:
+    exit = "exit"
+    ffi_error = "ffi-error"
+    message_echo = "message-echo"
+
+
+class AutoReplier:
+    def __init__(self, account, log, num_send, num_bigfiles, report_func):
+        self.account = account
+        self._log = log
+        self.report_func = report_func
+        self.num_send = num_send
+        self.num_bigfiles = num_bigfiles
+        self.current_sent = 0
+        self.addr = self.account.get_self_contact().addr
+
+        self._thread = threading.Thread(
+            name="Stats{}".format(self.account),
+            target=self.thread_stats
+        )
+        self._thread.setDaemon(True)
+        self._thread.start()
+
+    def log(self, message):
+        self._log("{} {}".format(self.addr, message))
+
+    def thread_stats(self):
+        # XXX later use, for now we just quit
+        return
+        while 1:
+            time.sleep(1.0)
+            break
+
+    @deltachat.account_hookimpl
+    def ac_incoming_message(self, message):
+        if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
+            return
+        message.create_chat()
+        message.mark_seen()
+        self.log("incoming message: {}".format(message))
+
+        self.current_sent += 1
+        # we are still alive, let's send a reply
+        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
+            message.chat.send_text("send big file as reply to: {}".format(message.text))
+            msg = message.chat.send_file(self.account.bigfile)
+        else:
+            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
+            assert msg.text
+        self.log("message-sent: {}".format(msg))
+        self.report_func(self, ReportType.message_echo)
+        if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
+            return
+
+    @deltachat.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self.log(ffi_event)
+        if ffi_event.name == "DC_EVENT_ERROR":
+            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_increation.py

@@ -6,27 +6,37 @@ import shutil
 import pytest
 from filecmp import cmp
 
-from conftest import wait_configuration_progress
 from deltachat import const
 
 
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev.data1 == chat_id
-    if msg_id is not None:
-        assert ev.data2 == msg_id
-    return ev.data2
+def wait_msg_delivered(account, msg_list):
+    """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
+    msg_list = list(msg_list)
+    while msg_list:
+        ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        msg_list.remove((ev.data1, ev.data2))
+
+
+def wait_msgs_changed(account, msgs_list):
+    """ wait for one or more MSGS_CHANGED events to match msgs_list contents. """
+    account.log("waiting for msgs_list={}".format(msgs_list))
+    msgs_list = list(msgs_list)
+    while msgs_list:
+        ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        for i, (data1, data2) in enumerate(msgs_list):
+            if ev.data1 == data1:
+                if data2 is None or ev.data2 == data2:
+                    del msgs_list[i]
+                    break
+        else:
+            account.log("waiting mismatch data1={} data2={}".format(data1, data2))
+    return ev.data1, ev.data2
 
 
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -35,13 +45,8 @@ class TestOnlineInCreation:
             chat.prepare_message_file(src.strpath)
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -53,15 +58,10 @@ class TestOnlineInCreation:
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
+        ac1, ac2 = acfactory.get_two_online_accounts()
 
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
-        wait_msgs_changed(ac1, 0, 0)  # why no chat id?
+        chat = ac1.create_chat(ac2)
+        wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
         orig = data.get_path("d.png")
@@ -70,19 +70,16 @@ class TestOnlineInCreation:
             fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
-        wait_msgs_changed(ac1, chat.id, prepared_original.id)
+        wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(c2)
-        wait_msgs_changed(ac1, 0, 0)  # why not chat id?
+        chat2.add_contact(ac2)
+        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them
         # is the one caused by forwarding
-        forwarded_id = wait_msgs_changed(ac1, chat2.id)
-        if forwarded_id == 0:
-            forwarded_id = wait_msgs_changed(ac1, chat2.id)
-            assert forwarded_id
+        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 
@@ -91,20 +88,18 @@ class TestOnlineInCreation:
         shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
-        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
-        lp.sec("expect the forwarded message to be sent now too")
-        wait_msgs_changed(ac1, chat2.id, forwarded_id)
+        lp.sec("check that both forwarded and original message are proper.")
+        wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
+
         fwd_msg = ac1.get_message_by_id(forwarded_id)
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
-        lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat.id
-        assert ev.data2 == prepared_original.id
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat2.id
-        assert ev.data2 == forwarded_id
+        lp.sec("wait for both messages to be delivered to SMTP")
+        wait_msg_delivered(ac1, [
+            (chat2.id, forwarded_id),
+            (chat.id, prepared_original.id)
+        ])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
         ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")

python/tests/test_lowlevel.py

@@ -1,75 +1,52 @@
 from __future__ import print_function
-from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
+
+from queue import Queue
+from deltachat import capi, cutil, const
 from deltachat import register_global_plugin
 from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
+# from deltachat.account import EventLogger
 
 
 def test_empty_context():
     ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
-    capi.lib.dc_close(ctx)
-
-
-def test_callback_None2int():
-    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
-    set_context_callback(ctx, lambda *args: None)
-    capi.lib.dc_close(ctx)
-    clear_context_callback(ctx)
+    capi.lib.dc_context_unref(ctx)
 
 
 def test_dc_close_events(tmpdir, acfactory):
     ac1 = acfactory.get_unconfigured_account()
 
     # register after_shutdown function
-    shutdowns = []
+    shutdowns = Queue()
 
     class ShutdownPlugin:
         @global_hookimpl
         def dc_account_after_shutdown(self, account):
             assert account._dc_context is None
-            shutdowns.append(account)
+            shutdowns.put(account)
     register_global_plugin(ShutdownPlugin())
     assert hasattr(ac1, "_dc_context")
     ac1.shutdown()
-    assert shutdowns == [ac1]
-
-    def find(info_string):
-        evlog = ac1._evtracker
-        while 1:
-            ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
-            data2 = ev.data2
-            if info_string in data2:
-                return
-            else:
-                print("skipping event", ev)
-
-    find("disconnecting inbox-thread")
-    find("disconnecting sentbox-thread")
-    find("disconnecting mvbox-thread")
-    find("disconnecting SMTP")
-    find("Database closed")
+    shutdowns.get(timeout=2)
 
 
 def test_wrong_db(tmpdir):
-    dc_context = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
     p = tmpdir.join("hello.db")
     # write an invalid database file
     p.write("x123" * 10)
-    assert not lib.dc_open(dc_context, p.strpath.encode("ascii"), ffi.NULL)
+
+    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
 
 
 def test_empty_blobdir(tmpdir):
+    db_fname = tmpdir.join("hello.db")
     # Apparently some client code expects this to be the same as passing NULL.
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
         lib.dc_context_unref,
     )
-    db_fname = tmpdir.join("hello.db")
-    assert lib.dc_open(ctx, db_fname.strpath.encode("ascii"), b"")
+    assert ctx != ffi.NULL
 
 
 def test_event_defines():
@@ -78,19 +55,22 @@ def test_event_defines():
 
 
 def test_sig():
-    sig = capi.lib.dc_get_event_signature_types
-    assert sig(const.DC_EVENT_INFO) == 2
-    assert sig(const.DC_EVENT_WARNING) == 2
-    assert sig(const.DC_EVENT_ERROR) == 2
-    assert sig(const.DC_EVENT_SMTP_CONNECTED) == 2
-    assert sig(const.DC_EVENT_IMAP_CONNECTED) == 2
-    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT) == 2
+    sig = capi.lib.dc_event_has_string_data
+    assert not sig(const.DC_EVENT_MSGS_CHANGED)
+    assert sig(const.DC_EVENT_INFO)
+    assert sig(const.DC_EVENT_WARNING)
+    assert sig(const.DC_EVENT_ERROR)
+    assert sig(const.DC_EVENT_SMTP_CONNECTED)
+    assert sig(const.DC_EVENT_IMAP_CONNECTED)
+    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
+    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
 
 
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
-    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    chat = ac1.create_chat_by_contact(contact1)
+
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
@@ -107,47 +87,18 @@ def test_get_special_message_id_returns_empty_message(acfactory):
 
 def test_provider_info_none():
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
 
 
-def test_get_info_closed():
-    ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
-    assert 'deltachat_core_version' in info
-    assert 'database_dir' not in info
-
-
 def test_get_info_open(tmpdir):
+    db_fname = tmpdir.join("test.db")
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
         lib.dc_context_unref,
     )
-    db_fname = tmpdir.join("test.db")
-    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
     assert 'deltachat_core_version' in info
     assert 'database_dir' in info
-
-
-def test_is_open_closed():
-    ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    assert lib.dc_is_open(ctx) == 0
-
-
-def test_is_open_actually_open(tmpdir):
-    ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    db_fname = tmpdir.join("test.db")
-    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
-    assert lib.dc_is_open(ctx) == 1

python/tox.ini

@@ -71,6 +71,8 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
+markers = 
+    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

rust-toolchain

@@ -1 +1 @@
-nightly-2020-03-12
+1.43.1

spec.md

@@ -1,10 +1,12 @@
-# Chat-over-Email specification
+# chat-mail specification
 
-Version 0.30.0
+Version: 0.32.0
+Status:  In-progress 
+Format:  [Semantic Line Breaks](https://sembr.org/)
 
-This document describes how emails can be used
-to implement typical messenger functions
-while staying compatible to existing MUAs.
+This document roughly describes how chat-mail 
+apps use the standard e-mail system 
+to implement typical messenger functions.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -30,17 +32,14 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
-If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD start with the characters `Chat:`
-and SHOULD be an excerpt of the message.
+the `Subject` header SHOULD be `Message from <sender name>`.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -58,7 +57,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Chat: Hello ...
+    Subject: Message from sender@domain
 
     Hello world!
 

src/aheader.rs

@@ -6,13 +6,15 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
+use deltachat_derive::*;
+
 use crate::contact::*;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};
 
 /// Possible values for encryption preference
-#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive)]
+#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive, Sqlx)]
 #[repr(u8)]
 pub enum EncryptPreference {
     NoPreference = 0,

src/blob.rs

@@ -2,16 +2,21 @@
 
 use std::ffi::OsStr;
 use std::fmt;
-use std::fs;
-use std::io::Write;
-use std::path::{Path, PathBuf};
+
+use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
+use async_std::{fs, io};
 
 use image::GenericImageView;
+use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::constants::AVATAR_SIZE;
+use crate::config::Config;
+use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::Event;
+use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -43,30 +48,35 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WriteFailure] is used when the file could not
     /// be written to.  You can expect [BlobError.cause] to contain an
     /// underlying error.
-    pub fn create(
+    pub async fn create(
         context: &'a Context,
         suggested_name: impl AsRef<str>,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
-        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext)?;
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
         file.write_all(data)
+            .await
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
     // Creates a new file, returning a tuple of the name and the handle.
-    fn create_new_file(dir: &Path, stem: &str, ext: &str) -> Result<(String, fs::File), BlobError> {
+    async fn create_new_file(
+        dir: &Path,
+        stem: &str,
+        ext: &str,
+    ) -> Result<(String, fs::File), BlobError> {
         let max_attempt = 15;
         let mut name = format!("{}{}", stem, ext);
         for attempt in 0..max_attempt {
@@ -75,6 +85,7 @@ impl<'a> BlobObject<'a> {
                 .create_new(true)
                 .write(true)
                 .open(&path)
+                .await
             {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
@@ -110,37 +121,41 @@ impl<'a> BlobObject<'a> {
     /// In addition to the errors in [BlobObject::create] the
     /// [BlobError::CopyFailure] is used when the data can not be
     /// copied.
-    pub fn create_and_copy(
+    pub async fn create_and_copy(
         context: &'a Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file = fs::File::open(src.as_ref()).map_err(|err| BlobError::CopyFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: String::from(""),
-            src: src.as_ref().to_path_buf(),
-            cause: err,
-        })?;
+        let mut src_file =
+            fs::File::open(src.as_ref())
+                .await
+                .map_err(|err| BlobError::CopyFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: String::from(""),
+                    src: src.as_ref().to_path_buf(),
+                    cause: err,
+                })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
-        let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
+        let (name, mut dst_file) =
+            BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
-        std::io::copy(&mut src_file, &mut dst_file).map_err(|err| {
+        if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
             {
                 // Attempt to remove the failed file, swallow errors resulting from that.
                 let path = context.get_blobdir().join(&name_for_err);
-                fs::remove_file(path).ok();
+                fs::remove_file(path).await.ok();
             }
-            BlobError::CopyFailure {
+            return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
-            }
-        })?;
+            });
+        }
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -158,14 +173,14 @@ impl<'a> BlobObject<'a> {
     /// This merely delegates to the [BlobObject::create_and_copy] and
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
-    pub fn new_from_path(
+    pub async fn new_from_path(
         context: &Context,
         src: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject, BlobError> {
+    ) -> std::result::Result<BlobObject<'_>, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
         } else {
-            BlobObject::create_and_copy(context, src)
+            BlobObject::create_and_copy(context, src).await
         }
     }
 
@@ -361,11 +376,73 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
     }
+
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
+            != Some((Viewtype::Image, "image/jpeg"))
+        {
+            return Ok(());
+        }
+
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
+
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
+        }
+
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
+    }
+
+    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exifreader = exif::Reader::new();
+        let exif = exifreader.read_from_container(&mut bufreader)?;
+        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
+            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
+            // we only use rotation, in practise, flipping is not used.
+            match orientation.value.get_uint(0) {
+                Some(3) => return Ok(180),
+                Some(6) => return Ok(90),
+                Some(8) => return Ok(270),
+                other => warn!(context, "exif orientation value ignored: {:?}", other),
+            }
+        }
+        Ok(0)
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -389,7 +466,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
@@ -418,58 +495,71 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[test]
-    fn test_create() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_create() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
         let fname = t.ctx.get_blobdir().join("foo");
-        let data = fs::read(fname).unwrap();
+        let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
         assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
     }
 
-    #[test]
-    fn test_lowercase_ext() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_lowercase_ext() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
-    #[test]
-    fn test_as_file_name() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_as_file_name() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
-    #[test]
-    fn test_as_rel_path() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_as_rel_path() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
-    #[test]
-    fn test_suffix() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_suffix() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
+        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
-    #[test]
-    fn test_create_dup() {
-        let t = dummy_context();
-        BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_create_dup() {
+        let t = TestContext::new().await;
+        BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         let foo_path = t.ctx.get_blobdir().join("foo.txt");
-        assert!(foo_path.exists());
-        BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
-        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+        assert!(foo_path.exists().await);
+        BlobObject::create(&t.ctx, "foo.txt", b"world")
+            .await
+            .unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
-                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
+                assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 assert!(name.starts_with("foo"));
@@ -478,17 +568,22 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_double_ext_preserved() {
-        let t = dummy_context();
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_double_ext_preserved() {
+        let t = TestContext::new().await;
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
+            .await
+            .unwrap();
         let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
-        assert!(foo_path.exists());
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"world").unwrap();
-        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+        assert!(foo_path.exists().await);
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
+            .await
+            .unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
-                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
+                assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 println!("{}", name);
@@ -498,55 +593,55 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_create_long_names() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_long_names() {
+        let t = TestContext::new().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t.ctx, &s, b"data").unwrap();
+        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
 
-    #[test]
-    fn test_create_and_copy() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_and_copy() {
+        let t = TestContext::new().await;
         let src = t.dir.path().join("src");
-        fs::write(&src, b"boo").unwrap();
-        let blob = BlobObject::create_and_copy(&t.ctx, &src).unwrap();
+        fs::write(&src, b"boo").await.unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
-        let data = fs::read(blob.to_abs_path()).unwrap();
+        let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).is_err());
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
         let whoops = t.ctx.get_blobdir().join("whoops");
-        assert!(!whoops.exists());
+        assert!(!whoops.exists().await);
     }
 
-    #[test]
-    fn test_create_from_path() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_from_path() {
+        let t = TestContext::new().await;
 
         let src_ext = t.dir.path().join("external");
-        fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        fs::write(&src_ext, b"boo").await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
-        let data = fs::read(blob.to_abs_path()).unwrap();
+        let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let src_int = t.ctx.get_blobdir().join("internal");
-        fs::write(&src_int, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).unwrap();
+        fs::write(&src_int, b"boo").await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
-        let data = fs::read(blob.to_abs_path()).unwrap();
+        let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
     }
-    #[test]
-    fn test_create_from_name_long() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_from_name_long() {
+        let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
-        fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        fs::write(&src_ext, b"boo").await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -565,8 +660,40 @@ mod tests {
 
     #[test]
     fn test_sanitise_name() {
-        let (_, ext) =
+        let (stem, ext) =
             BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
         assert_eq!(ext, ".txt");
+        assert!(!stem.is_empty());
+
+        // the extensions are kept together as between stem and extension a number may be added -
+        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
+        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
+        assert_eq!(stem, "wot");
+        assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
+        assert_eq!(stem, "");
+        assert_eq!(ext, ".foo.bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
+        assert!(stem.contains("foo"));
+        assert!(!stem.contains("?"));
+        assert_eq!(ext, ".bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("no-extension");
+        assert_eq!(stem, "no-extension");
+        assert_eq!(ext, "");
+
+        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
+        assert_eq!(ext, ".c");
+        assert!(!stem.contains("path"));
+        assert!(!stem.contains("ignored"));
+        assert!(stem.contains("this"));
+        assert!(stem.contains("forbidden"));
+        assert!(!stem.contains("/"));
+        assert!(!stem.contains("\\"));
+        assert!(!stem.contains(":"));
+        assert!(!stem.contains("*"));
+        assert!(!stem.contains("?"));
     }
 }

src/chatlist.rs

@@ -86,33 +86,28 @@ impl Chatlist {
     ///     are returned.
     /// `query_contact_id`: An optional contact ID for filtering the list. Only chats including this contact ID
     ///     are returned.
-    pub fn try_load(
+    pub async fn try_load(
         context: &Context,
         listflags: usize,
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
+        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
+        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
+        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
+        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
+
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get hidden to avoid reloading the same chatlist.
-        if let Err(err) = hide_device_expired_messages(context) {
+        // messages get deleted to avoid reloading the same chatlist.
+        if let Err(err) = delete_device_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
         let mut add_archived_link_item = false;
 
-        let process_row = |row: &rusqlite::Row| {
-            let chat_id: ChatId = row.get(0)?;
-            let msg_id: MsgId = row.get(1).unwrap_or_default();
-            Ok((chat_id, msg_id))
-        };
-
-        let process_rows = |rows: rusqlite::MappedRows<_>| {
-            rows.collect::<std::result::Result<Vec<_>, _>>()
-                .map_err(Into::into)
-        };
-
-        let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+        let skip_id = if flag_for_forwarding {
             chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+                .await
                 .unwrap_or_default()
                 .0
         } else {
@@ -136,110 +131,115 @@ impl Chatlist {
         // shown at all permanent in the chatlist.
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
-            context.sql.query_map(
+            context.sql.query_rows(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
-                process_row,
-                process_rows,
-            )?
-        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
+                 ORDER BY c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsx![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+            ).await?
+        } else if flag_archived_only {
             // show archived chats
             // (this includes the archived device-chat; we could skip it,
             // however, then the number of archived chats do not match, which might be even more irritating.
             // and adapting the number requires larger refactorings and seems not to be worth the effort)
-            context.sql.query_map(
-                "SELECT c.id, m.id
+            context
+                .sql
+                .query_rows(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft],
-                process_row,
-                process_rows,
-            )?
+                    paramsx![MessageState::OutDraft],
+                )
+                .await?
         } else if let Some(query) = query {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
 
             // allow searching over special names that may change at any time
             // when the ui calls set_stock_translation()
-            if let Err(err) = update_special_chat_names(context) {
+            if let Err(err) = update_special_chat_names(context).await {
                 warn!(context, "cannot update special chat names: {:?}", err)
             }
 
             let str_like_cmd = format!("%{}%", query);
-            context.sql.query_map(
-                "SELECT c.id, m.id
+            context
+                .sql
+                .query_rows(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND c.name LIKE ?3
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, str_like_cmd],
-                process_row,
-                process_rows,
-            )?
+                    paramsx![MessageState::OutDraft, skip_id, str_like_cmd],
+                )
+                .await?
         } else {
             //  show normal chatlist
-            let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+            let sort_id_up = if flag_for_forwarding {
                 chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+                    .await
                     .unwrap_or_default()
                     .0
             } else {
                 ChatId::new(0)
             };
-            let mut ids = context.sql.query_map(
+            let mut ids = context.sql.query_rows(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND NOT c.archived=?3
                  GROUP BY c.id
                  ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
-                process_row,
-                process_rows,
-            )?;
-            if 0 == listflags & DC_GCL_NO_SPECIALS {
-                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    if 0 == listflags & DC_GCL_FOR_FORWARDING {
+                paramsx![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+            ).await?;
+
+            if !flag_no_specials {
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context).await
+                {
+                    if !flag_for_forwarding {
                         ids.insert(
                             0,
                             (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
@@ -251,8 +251,8 @@ impl Chatlist {
             ids
         };
 
-        if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
-            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
+        if add_archived_link_item && dc_get_archived_cnt(context).await > 0 {
+            if ids.is_empty() && flag_add_alldone_hint {
                 ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
             ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
@@ -305,7 +305,7 @@ impl Chatlist {
     /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
     /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
     //    0 if not applicable.
-    pub fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
+    pub async fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
@@ -323,7 +323,7 @@ impl Chatlist {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) {
+        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
@@ -332,11 +332,11 @@ impl Chatlist {
 
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
-                lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
             }
 
             Some(lastmsg)
@@ -348,9 +348,15 @@ impl Chatlist {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(context.stock_str(StockMessage::NoMessages).to_string());
+            ret.text2 = Some(
+                context
+                    .stock_str(StockMessage::NoMessages)
+                    .await
+                    .to_string(),
+            );
         } else {
-            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context);
+            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
+                .await;
         }
 
         ret
@@ -362,34 +368,38 @@ impl Chatlist {
 }
 
 /// Returns the number of archived chats
-pub fn dc_get_archived_cnt(context: &Context) -> u32 {
-    context
+pub async fn dc_get_archived_cnt(context: &Context) -> u32 {
+    let v: i32 = context
         .sql
-        .query_get_value(
-            context,
+        .query_value(
             "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
-            params![],
+            paramsx![],
         )
-        .unwrap_or_default()
+        .await
+        .unwrap_or_default();
+    v as u32
 }
 
-fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
     // We have an index over the state-column, this should be
     // sufficient as there are typically only few fresh messages.
-    context.sql.query_get_value(
-        context,
-        concat!(
-            "SELECT m.id",
-            " FROM msgs m",
-            " LEFT JOIN chats c",
-            "        ON c.id=m.chat_id",
-            " WHERE m.state=10",
-            "   AND m.hidden=0",
-            "   AND c.blocked=2",
-            " ORDER BY m.timestamp DESC, m.id DESC;"
-        ),
-        params![],
-    )
+    context
+        .sql
+        .query_value(
+            r#"
+SELECT m.id
+  FROM msgs m
+  LEFT JOIN chats c
+        ON c.id=m.chat_id
+ WHERE m.state=10
+   AND m.hidden=0
+   AND c.blocked=2
+ ORDER BY m.timestamp DESC, m.id DESC;
+"#,
+            paramsx![],
+        )
+        .await
+        .ok()
 }
 
 #[cfg(test)]
@@ -398,15 +408,21 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[test]
-    fn test_try_load() {
-        let t = dummy_context();
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat").unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat").unwrap();
+    #[async_std::test]
+    async fn test_try_load() {
+        let t = TestContext::new().await;
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+            .await
+            .unwrap();
+        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
+            .await
+            .unwrap();
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
+            .await
+            .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -415,77 +431,102 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg));
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
             .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .await
             .ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 1);
     }
 
-    #[test]
-    fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context();
-        t.ctx.update_device_chats().unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+    #[async_std::test]
+    async fn test_sort_self_talk_up_on_forward() {
+        let t = TestContext::new().await;
+        t.ctx.update_device_chats().await.unwrap();
+        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+            .await
+            .unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
         assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
+            .await
+            .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
         assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .await
             .unwrap()
             .is_self_talk());
     }
 
-    #[test]
-    fn test_search_special_chat_names() {
-        let t = dummy_context();
-        t.ctx.update_device_chats().unwrap();
+    #[async_std::test]
+    async fn test_search_special_chat_names() {
+        let t = TestContext::new().await;
+        t.ctx.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 0);
 
         t.ctx
             .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+            .await
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+            .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
         assert_eq!(chats.len(), 1);
 
         t.ctx
             .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+            .await
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+            .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
         assert_eq!(chats.len(), 1);
     }
 
-    #[test]
-    fn test_get_summary_unwrap() {
-        let t = dummy_context();
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+    #[async_std::test]
+    async fn test_get_summary_unwrap() {
+        let t = TestContext::new().await;
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+            .await
+            .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg));
+        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None);
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t.ctx, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/config.rs

@@ -9,11 +9,9 @@ use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::events::Event;
-use crate::job::*;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
-use rusqlite::NO_PARAMS;
 
 /// The available configuration keys.
 #[derive(
@@ -65,6 +63,9 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
+    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
+    MediaQuality,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -103,6 +104,9 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
+    ConfiguredInboxFolder,
+    ConfiguredMvboxFolder,
+    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -116,17 +120,21 @@ pub enum Config {
 }
 
 impl Context {
+    pub async fn config_exists(&self, key: Config) -> bool {
+        self.sql.get_raw_config(key).await.is_some()
+    }
+
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
-    pub fn get_config(&self, key: Config) -> Option<String> {
+    pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_raw_config(self, key);
+                let rel_path = self.sql.get_raw_config(key).await;
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_raw_config(self, key),
+            _ => self.sql.get_raw_config(key).await,
         };
 
         if value.is_some() {
@@ -135,27 +143,29 @@ impl Context {
 
         // Default values
         match key {
-            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).into_owned()),
+            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
 
-    pub fn get_config_int(&self, key: Config) -> i32 {
+    pub async fn get_config_int(&self, key: Config) -> i32 {
         self.get_config(key)
+            .await
             .and_then(|s| s.parse().ok())
             .unwrap_or_default()
     }
 
-    pub fn get_config_bool(&self, key: Config) -> bool {
-        self.get_config_int(key) != 0
+    pub async fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key).await != 0
     }
 
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
     /// at once, `Some(x)` means delete after `x` seconds.
-    pub fn get_config_delete_server_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteServerAfter) {
+    pub async fn get_config_delete_server_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteServerAfter).await {
             0 => None,
             1 => Some(0),
             x => Some(x as i64),
@@ -166,8 +176,8 @@ impl Context {
     ///
     /// `None` means never delete the message, `Some(x)` means delete
     /// after `x` seconds.
-    pub fn get_config_delete_device_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteDeviceAfter) {
+    pub async fn get_config_delete_device_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteDeviceAfter).await {
             0 => None,
             x => Some(x as i64),
         }
@@ -175,57 +185,46 @@ impl Context {
 
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
+    pub async fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
             Config::Selfavatar => {
                 self.sql
-                    .execute("UPDATE contacts SET selfavatar_sent=0;", NO_PARAMS)?;
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", paramsx![])
+                    .await?;
                 self.sql
-                    .set_raw_config_bool(self, "attach_selfavatar", true)?;
+                    .set_raw_config_bool(self, "attach_selfavatar", true)
+                    .await?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(&self, value)?;
+                        let blob = BlobObject::new_from_path(&self, value).await?;
                         blob.recode_to_avatar_size(self)?;
-                        self.sql.set_raw_config(self, key, Some(blob.as_name()))
+                        self.sql
+                            .set_raw_config(self, key, Some(blob.as_name()))
+                            .await
                     }
-                    None => self.sql.set_raw_config(self, key, None),
+                    None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
-            Config::InboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_inbox_idle(self);
-                ret
-            }
-            Config::SentboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_sentbox_idle(self);
-                ret
-            }
-            Config::MvboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_mvbox_idle(self);
-                ret
-            }
             Config::Selfstatus => {
-                let def = self.stock_str(StockMessage::StatusLine);
+                let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
                     value
                 };
 
-                self.sql.set_raw_config(self, key, val)
+                self.sql.set_raw_config(self, key, val).await
             }
             Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
-                self.call_cb(Event::MsgsChanged {
+                self.emit_event(Event::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 });
                 ret
             }
-            _ => self.sql.set_raw_config(self, key, value),
+            _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
 }
@@ -248,9 +247,11 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::constants;
     use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
     use image::GenericImageView;
+    use num_traits::FromPrimitive;
     use std::fs::File;
     use std::io::Write;
 
@@ -271,9 +272,9 @@ mod tests {
         assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
     }
 
-    #[test]
-    fn test_selfavatar_outside_blobdir() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_selfavatar_outside_blobdir() {
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         File::create(&avatar_src)
@@ -281,13 +282,14 @@ mod tests {
             .write_all(avatar_bytes)
             .unwrap();
         let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists());
+        assert!(!avatar_blob.exists().await);
         t.ctx
             .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
             .unwrap();
-        assert!(avatar_blob.exists());
+        assert!(avatar_blob.exists().await);
         assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
@@ -299,9 +301,9 @@ mod tests {
         assert_eq!(img.height(), AVATAR_SIZE);
     }
 
-    #[test]
-    fn test_selfavatar_in_blobdir() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_selfavatar_in_blobdir() {
+        let t = TestContext::new().await;
         let avatar_src = t.ctx.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
@@ -315,12 +317,57 @@ mod tests {
 
         t.ctx
             .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
             .unwrap();
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
         assert_eq!(img.width(), AVATAR_SIZE);
         assert_eq!(img.height(), AVATAR_SIZE);
     }
+
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists().await);
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+    }
+
+    #[async_std::test]
+    async fn test_media_quality_config_option() {
+        let t = TestContext::new().await;
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        assert_eq!(media_quality, 0);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Balanced);
+
+        t.ctx
+            .set_config(Config::MediaQuality, Some("1"))
+            .await
+            .unwrap();
+
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        assert_eq!(media_quality, 1);
+        assert_eq!(constants::MediaQuality::Worse as i32, 1);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Worse);
+    }
 }

src/configure/auto_mozilla.rs

@@ -94,12 +94,12 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
     }
 }
 
-pub fn moz_autoconfigure(
+pub async fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
 ) -> Result<LoginParam, Error> {
-    let xml_raw = read_url(context, url)?;
+    let xml_raw = read_url(context, url).await?;
 
     let res = parse_xml(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/auto_outlook.rs

@@ -112,7 +112,7 @@ fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     Ok(res)
 }
 
-pub fn outlk_autodiscover(
+pub async fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
@@ -120,7 +120,7 @@ pub fn outlk_autodiscover(
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
-        let xml_raw = read_url(context, &url)?;
+        let xml_raw = read_url(context, &url).await?;
         let res = parse_xml(&xml_raw);
         if let Err(err) = &res {
             warn!(context, "{}", err);

src/configure/read_url.rs

@@ -3,17 +3,13 @@ use crate::context::Context;
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("URL request error")]
-    GetError(#[from] reqwest::Error),
+    GetError(surf::Error),
 }
 
-pub fn read_url(context: &Context, url: &str) -> Result<String, Error> {
+pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     info!(context, "Requesting URL {}", url);
 
-    match reqwest::blocking::Client::new()
-        .get(url)
-        .send()
-        .and_then(|res| res.text())
-    {
+    match surf::get(url).recv_string().await {
         Ok(res) => Ok(res),
         Err(err) => {
             info!(context, "Can\'t read URL {}", url);

src/constants.rs

@@ -1,7 +1,6 @@
 //! # Constants
 #![allow(dead_code)]
 
-use deltachat_derive::*;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
@@ -25,12 +24,11 @@ const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
     Eq,
     FromPrimitive,
     ToPrimitive,
-    FromSql,
-    ToSql,
     Serialize,
     Deserialize,
+    sqlx::Type,
 )]
-#[repr(u8)]
+#[repr(i32)]
 pub enum Blocked {
     Not = 0,
     Manually = 1,
@@ -43,8 +41,8 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
-#[repr(u8)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
+#[repr(i32)]
 pub enum ShowEmails {
     Off = 0,
     AcceptedContacts = 1,
@@ -57,7 +55,20 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
+#[repr(i32)]
+pub enum MediaQuality {
+    Balanced = 0,
+    Worse = 1,
+}
+
+impl Default for MediaQuality {
+    fn default() -> Self {
+        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
+    }
+}
+
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -114,13 +125,12 @@ pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
     Eq,
     FromPrimitive,
     ToPrimitive,
-    FromSql,
-    ToSql,
     IntoStaticStr,
     Serialize,
     Deserialize,
+    sqlx::Type,
 )]
-#[repr(u32)]
+#[repr(i32)]
 pub enum Chattype {
     Undefined = 0,
     Single = 100,
@@ -214,6 +224,10 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
+// max. width/height of images
+pub const BALANCED_IMAGE_SIZE: u32 = 1280;
+pub const WORSE_IMAGE_SIZE: u32 = 640;
+
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
@@ -226,10 +240,9 @@ pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
     Eq,
     FromPrimitive,
     ToPrimitive,
-    FromSql,
-    ToSql,
     Serialize,
     Deserialize,
+    sqlx::Type,
 )]
 #[repr(i32)]
 pub enum Viewtype {
@@ -314,8 +327,6 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
 const DC_STR_DRAFT: usize = 3;
-const DC_STR_MEMBER: usize = 4;
-const DC_STR_CONTACT: usize = 6;
 const DC_STR_VOICEMESSAGE: usize = 7;
 const DC_STR_DEADDROP: usize = 8;
 const DC_STR_IMAGE: usize = 9;
@@ -347,7 +358,6 @@ const DC_STR_ARCHIVEDCHATS: usize = 40;
 const DC_STR_STARREDMSGS: usize = 41;
 const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
 const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_SELFTALK_SUBTITLE: usize = 50;
 const DC_STR_CANNOT_LOGIN: usize = 60;
 const DC_STR_SERVER_RESPONSE: usize = 61;
 const DC_STR_MSGACTIONBYUSER: usize = 62;

src/context.rs

@@ -1,68 +1,70 @@
 //! Context module
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
-use std::path::{Path, PathBuf};
-use std::sync::{Arc, Condvar, Mutex, RwLock};
+use std::ops::Deref;
+
+use async_std::path::{Path, PathBuf};
+use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
 
 use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
+use crate::dc_tools::duration_to_str;
 use crate::error::*;
-use crate::events::Event;
-use crate::imap::*;
-use crate::job::*;
-use crate::job_thread::JobThread;
-use crate::key::Key;
+use crate::events::{Event, EventEmitter, Events};
+use crate::job::{self, Action};
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
 use crate::message::{self, Message, MessengerMessage, MsgId};
 use crate::param::Params;
-use crate::smtp::Smtp;
+use crate::scheduler::Scheduler;
 use crate::sql::Sql;
+use std::time::SystemTime;
 
-/// Callback function type for [Context]
-///
-/// # Parameters
-///
-/// * `context` - The context object as returned by [Context::new].
-/// * `event` - One of the [Event] items.
-/// * `data1` - Depends on the event parameter, see [Event].
-/// * `data2` - Depends on the event parameter, see [Event].
-pub type ContextCallback = dyn Fn(&Context, Event) -> () + Send + Sync;
-
-#[derive(DebugStub)]
+#[derive(Clone, Debug)]
 pub struct Context {
-    /// Database file path
-    dbfile: PathBuf,
-    /// Blob directory path
-    blobdir: PathBuf,
-    pub sql: Sql,
-    pub perform_inbox_jobs_needed: Arc<RwLock<bool>>,
-    pub probe_imap_network: Arc<RwLock<bool>>,
-    pub inbox_thread: Arc<RwLock<JobThread>>,
-    pub sentbox_thread: Arc<RwLock<JobThread>>,
-    pub mvbox_thread: Arc<RwLock<JobThread>>,
-    pub smtp: Arc<Mutex<Smtp>>,
-    pub smtp_state: Arc<(Mutex<SmtpState>, Condvar)>,
-    pub oauth2_critical: Arc<Mutex<()>>,
-    #[debug_stub = "Callback"]
-    cb: Box<ContextCallback>,
-    pub os_name: Option<String>,
-    pub cmdline_sel_chat_id: Arc<RwLock<ChatId>>,
-    pub(crate) bob: Arc<RwLock<BobStatus>>,
-    pub last_smeared_timestamp: RwLock<i64>,
-    pub running_state: Arc<RwLock<RunningState>>,
-    /// Mutex to avoid generating the key for the user more than once.
-    pub generating_key_mutex: Mutex<()>,
-    pub translated_stockstrings: RwLock<HashMap<usize, String>>,
+    pub(crate) inner: Arc<InnerContext>,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+impl Deref for Context {
+    type Target = InnerContext;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+#[derive(Debug)]
+pub struct InnerContext {
+    /// Database file path
+    pub(crate) dbfile: PathBuf,
+    /// Blob directory path
+    pub(crate) blobdir: PathBuf,
+    pub(crate) sql: Sql,
+    pub(crate) os_name: Option<String>,
+    pub(crate) bob: RwLock<BobStatus>,
+    pub(crate) last_smeared_timestamp: RwLock<i64>,
+    pub(crate) running_state: RwLock<RunningState>,
+    /// Mutex to avoid generating the key for the user more than once.
+    pub(crate) generating_key_mutex: Mutex<()>,
+    /// Mutex to enforce only a single running oauth2 is running.
+    pub(crate) oauth2_mutex: Mutex<()>,
+    pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
+    pub(crate) events: Events,
+
+    pub(crate) scheduler: RwLock<Scheduler>,
+
+    creation_time: SystemTime,
+}
+
+#[derive(Debug)]
 pub struct RunningState {
     pub ongoing_running: bool,
     shall_stop_ongoing: bool,
+    cancel_sender: Option<Sender<()>>,
 }
 
 /// Return some info about deltachat-core
@@ -71,10 +73,14 @@ pub struct RunningState {
 /// actual keys and their values which will be present are not
 /// guaranteed.  Calling [Context::get_info] also includes information
 /// about the context on top of the information here.
-pub fn get_info() -> HashMap<&'static str, String> {
-    let mut res = HashMap::new();
+pub fn get_info() -> BTreeMap<&'static str, String> {
+    let mut res = BTreeMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
-    res.insert("sqlite_version", rusqlite::version().to_string());
+
+    let version =
+        String::from_utf8(libsqlite3_sys::SQLITE_VERSION.to_vec()).expect("invalid version");
+    res.insert("sqlite_version", version);
+
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
     res.insert("level", "awesome".into());
     res
@@ -82,72 +88,91 @@ pub fn get_info() -> HashMap<&'static str, String> {
 
 impl Context {
     /// Creates new context.
-    pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
-        pretty_env_logger::try_init_timed().ok();
-
+    pub async fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         let blobdir = dbfile.with_file_name(blob_fname);
-        if !blobdir.exists() {
-            std::fs::create_dir_all(&blobdir)?;
+        if !blobdir.exists().await {
+            async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(cb, os_name, dbfile, blobdir)
+        Context::with_blobdir(os_name, dbfile, blobdir).await
     }
 
-    pub fn with_blobdir(
-        cb: Box<ContextCallback>,
+    pub async fn with_blobdir(
         os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
     ) -> Result<Context> {
         ensure!(
-            blobdir.is_dir(),
+            blobdir.is_dir().await,
             "Blobdir does not exist: {}",
             blobdir.display()
         );
-        let ctx = Context {
+
+        let inner = InnerContext {
             blobdir,
             dbfile,
-            cb,
             os_name: Some(os_name),
-            running_state: Arc::new(RwLock::new(Default::default())),
+            running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
-            smtp: Arc::new(Mutex::new(Smtp::new())),
-            smtp_state: Arc::new((Mutex::new(Default::default()), Condvar::new())),
-            oauth2_critical: Arc::new(Mutex::new(())),
-            bob: Arc::new(RwLock::new(Default::default())),
+            bob: RwLock::new(Default::default()),
             last_smeared_timestamp: RwLock::new(0),
-            cmdline_sel_chat_id: Arc::new(RwLock::new(ChatId::new(0))),
-            inbox_thread: Arc::new(RwLock::new(JobThread::new(
-                "INBOX",
-                "configured_inbox_folder",
-                Imap::new(),
-            ))),
-            sentbox_thread: Arc::new(RwLock::new(JobThread::new(
-                "SENTBOX",
-                "configured_sentbox_folder",
-                Imap::new(),
-            ))),
-            mvbox_thread: Arc::new(RwLock::new(JobThread::new(
-                "MVBOX",
-                "configured_mvbox_folder",
-                Imap::new(),
-            ))),
-            probe_imap_network: Arc::new(RwLock::new(false)),
-            perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
+            oauth2_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
+            events: Events::default(),
+            scheduler: RwLock::new(Scheduler::Stopped),
+            creation_time: std::time::SystemTime::now(),
         };
 
-        ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, false),
-            "Failed opening sqlite database"
-        );
+        let ctx = Context {
+            inner: Arc::new(inner),
+        };
+
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
 
+    /// Starts the IO scheduler.
+    pub async fn start_io(&self) {
+        info!(self, "starting IO");
+        if self.is_io_running().await {
+            info!(self, "IO is already running");
+            return;
+        }
+
+        {
+            let l = &mut *self.inner.scheduler.write().await;
+            l.start(self.clone()).await;
+        }
+    }
+
+    /// Returns if the IO scheduler is running.
+    pub async fn is_io_running(&self) -> bool {
+        self.inner.is_io_running().await
+    }
+
+    /// Stops the IO scheduler.
+    pub async fn stop_io(&self) {
+        info!(self, "stopping IO");
+        if !self.is_io_running().await {
+            info!(self, "IO is not running");
+            return;
+        }
+
+        self.inner.stop_io().await;
+    }
+
+    /// Returns a reference to the underlying SQL instance.
+    ///
+    /// Warning: this is only here for testing, not part of the public API.
+    #[cfg(feature = "internals")]
+    pub fn sql(&self) -> &Sql {
+        &self.inner.sql
+    }
+
     /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
         self.dbfile.as_path()
@@ -158,49 +183,57 @@ impl Context {
         self.blobdir.as_path()
     }
 
-    pub fn call_cb(&self, event: Event) {
-        (*self.cb)(self, event);
+    /// Emits a single event.
+    pub fn emit_event(&self, event: Event) {
+        self.events.emit(event);
     }
 
-    /*******************************************************************************
-     * Ongoing process allocation/free/check
-     ******************************************************************************/
+    /// Get the next queued event.
+    pub fn get_event_emitter(&self) -> EventEmitter {
+        self.events.get_emitter()
+    }
 
-    pub fn alloc_ongoing(&self) -> bool {
-        if self.has_ongoing() {
-            warn!(self, "There is already another ongoing process running.",);
+    // Ongoing process allocation/free/check
 
-            false
-        } else {
-            let s_a = self.running_state.clone();
-            let mut s = s_a.write().unwrap();
-
-            s.ongoing_running = true;
-            s.shall_stop_ongoing = false;
-
-            true
+    pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
+        if self.has_ongoing().await {
+            bail!("There is already another ongoing process running.");
         }
+
+        let s_a = &self.running_state;
+        let mut s = s_a.write().await;
+
+        s.ongoing_running = true;
+        s.shall_stop_ongoing = false;
+        let (sender, receiver) = channel(1);
+        s.cancel_sender = Some(sender);
+
+        Ok(receiver)
     }
 
-    pub fn free_ongoing(&self) {
-        let s_a = self.running_state.clone();
-        let mut s = s_a.write().unwrap();
+    pub async fn free_ongoing(&self) {
+        let s_a = &self.running_state;
+        let mut s = s_a.write().await;
 
         s.ongoing_running = false;
         s.shall_stop_ongoing = true;
+        s.cancel_sender.take();
     }
 
-    pub fn has_ongoing(&self) -> bool {
-        let s_a = self.running_state.clone();
-        let s = s_a.read().unwrap();
+    pub async fn has_ongoing(&self) -> bool {
+        let s_a = &self.running_state;
+        let s = s_a.read().await;
 
         s.ongoing_running || !s.shall_stop_ongoing
     }
 
     /// Signal an ongoing process to stop.
-    pub fn stop_ongoing(&self) {
-        let s_a = self.running_state.clone();
-        let mut s = s_a.write().unwrap();
+    pub async fn stop_ongoing(&self) {
+        let s_a = &self.running_state;
+        let mut s = s_a.write().await;
+        if let Some(cancel) = s.cancel_sender.take() {
+            cancel.send(()).await;
+        }
 
         if s.ongoing_running && !s.shall_stop_ongoing {
             info!(self, "Signaling the ongoing process to stop ASAP.",);
@@ -210,69 +243,71 @@ impl Context {
         };
     }
 
-    pub fn shall_stop_ongoing(&self) -> bool {
-        self.running_state
-            .clone()
-            .read()
-            .unwrap()
-            .shall_stop_ongoing
+    pub async fn shall_stop_ongoing(&self) -> bool {
+        self.running_state.read().await.shall_stop_ongoing
     }
 
     /*******************************************************************************
      * UI chat/message related API
      ******************************************************************************/
 
-    pub fn get_info(&self) -> HashMap<&'static str, String> {
+    pub async fn get_info(&self) -> BTreeMap<&'static str, String> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "");
-        let l2 = LoginParam::from_database(self, "configured_");
-        let displayname = self.get_config(Config::Displayname);
-        let chats = get_chat_cnt(self) as usize;
-        let real_msgs = message::get_real_msg_cnt(self) as usize;
-        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self) as usize;
-        let contacts = Contact::get_real_cnt(self) as usize;
-        let is_configured = self.get_config_int(Config::Configured);
+        let l = LoginParam::from_database(self, "").await;
+        let l2 = LoginParam::from_database(self, "configured_").await;
+        let displayname = self.get_config(Config::Displayname).await;
+        let chats = get_chat_cnt(self).await as usize;
+        let real_msgs = message::get_real_msg_cnt(self).await as usize;
+        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self).await as usize;
+        let contacts = Contact::get_real_cnt(self).await as usize;
+        let is_configured = self.get_config_int(Config::Configured).await;
         let dbversion = self
             .sql
-            .get_raw_config_int(self, "dbversion")
+            .get_raw_config_int("dbversion")
+            .await
             .unwrap_or_default();
+        let journal_mode = self
+            .sql
+            .query_value("PRAGMA journal_mode;", paramsx![])
+            .await
+            .unwrap_or_else(|_| "unknown".to_string());
+        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await;
+        let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await;
+        let bcc_self = self.get_config_int(Config::BccSelf).await;
 
-        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
-        let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
-        let bcc_self = self.get_config_int(Config::BccSelf);
+        let prv_key_cnt: Option<i32> = self
+            .sql
+            .query_value("SELECT COUNT(*) FROM keypairs;", paramsx![])
+            .await
+            .ok();
 
-        let prv_key_cnt: Option<isize> =
-            self.sql
-                .query_get_value(self, "SELECT COUNT(*) FROM keypairs;", rusqlite::NO_PARAMS);
-
-        let pub_key_cnt: Option<isize> = self.sql.query_get_value(
-            self,
-            "SELECT COUNT(*) FROM acpeerstates;",
-            rusqlite::NO_PARAMS,
-        );
-
-        let fingerprint_str = if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql) {
-            key.fingerprint()
-        } else {
-            "<Not yet calculated>".into()
+        let pub_key_cnt: Option<i32> = self
+            .sql
+            .query_value("SELECT COUNT(*) FROM acpeerstates;", paramsx![])
+            .await
+            .ok();
+        let fingerprint_str = match SignedPublicKey::load_self(self).await {
+            Ok(key) => key.fingerprint().hex(),
+            Err(err) => format!("<key failure: {}>", err),
         };
 
-        let inbox_watch = self.get_config_int(Config::InboxWatch);
-        let sentbox_watch = self.get_config_int(Config::SentboxWatch);
-        let mvbox_watch = self.get_config_int(Config::MvboxWatch);
-        let mvbox_move = self.get_config_int(Config::MvboxMove);
+        let inbox_watch = self.get_config_int(Config::InboxWatch).await;
+        let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
+        let mvbox_move = self.get_config_int(Config::MvboxMove).await;
         let folders_configured = self
             .sql
-            .get_raw_config_int(self, "folders_configured")
+            .get_raw_config_int("folders_configured")
+            .await
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
+            .get_config(Config::ConfiguredSentboxFolder)
+            .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
+            .get_config(Config::ConfiguredMvboxFolder)
+            .await
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
@@ -282,11 +317,13 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
+        res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
             "selfavatar",
             self.get_config(Config::Selfavatar)
+                .await
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
@@ -312,42 +349,38 @@ impl Context {
         );
         res.insert("fingerprint", fingerprint_str);
 
+        let elapsed = self.creation_time.elapsed();
+        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
+
         res
     }
 
-    pub fn get_fresh_msgs(&self) -> Vec<MsgId> {
-        let show_deaddrop = 0;
+    pub async fn get_fresh_msgs(&self) -> Vec<MsgId> {
+        let show_deaddrop: i32 = 0;
         self.sql
-            .query_map(
-                concat!(
-                    "SELECT m.id",
-                    " FROM msgs m",
-                    " LEFT JOIN contacts ct",
-                    "        ON m.from_id=ct.id",
-                    " LEFT JOIN chats c",
-                    "        ON m.chat_id=c.id",
-                    " WHERE m.state=?",
-                    "   AND m.hidden=0",
-                    "   AND m.chat_id>?",
-                    "   AND ct.blocked=0",
-                    "   AND (c.blocked=0 OR c.blocked=?)",
-                    " ORDER BY m.timestamp DESC,m.id DESC;"
-                ),
-                &[10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
-                |row| row.get::<_, MsgId>(0),
-                |rows| {
-                    let mut ret = Vec::new();
-                    for row in rows {
-                        ret.push(row?);
-                    }
-                    Ok(ret)
-                },
+            .query_values(
+                r#"
+SELECT m.id
+ FROM msgs m
+ LEFT JOIN contacts ct
+        ON m.from_id=ct.id
+ LEFT JOIN chats c
+        ON m.chat_id=c.id
+ WHERE m.state=?
+   AND m.hidden=0
+   AND m.chat_id>?
+   AND ct.blocked=0
+   AND (c.blocked=0 OR c.blocked=?)
+ ORDER BY m.timestamp DESC,m.id DESC;
+"#,
+                paramsx![10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
             )
+            .await
             .unwrap_or_default()
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -356,82 +389,64 @@ impl Context {
         let strLikeBeg = format!("{}%", real_query);
 
         let query = if !chat_id.is_unset() {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " WHERE m.chat_id=?",
-                "   AND m.hidden=0",
-                "   AND ct.blocked=0",
-                "   AND (txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp,m.id;"
-            )
+            r#"
+SELECT m.id
+ FROM msgs 
+ LEFT JOIN contacts ct
+        ON m.from_id=ct.id
+ WHERE m.chat_id=?
+   AND m.hidden=0
+   AND ct.blocked=0
+   AND (txt LIKE ? OR ct.name LIKE ?)
+ ORDER BY m.timestamp,m.id;
+"#
         } else {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " LEFT JOIN chats c",
-                "        ON m.chat_id=c.id",
-                " WHERE m.chat_id>9",
-                "   AND m.hidden=0",
-                "   AND (c.blocked=0 OR c.blocked=?)",
-                "   AND ct.blocked=0",
-                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp DESC,m.id DESC;"
-            )
+            r#"
+SELECT m.id
+ FROM msgs m
+ LEFT JOIN contacts ct
+        ON m.from_id=ct.id
+ LEFT JOIN chats c
+        ON m.chat_id=c.id
+ WHERE m.chat_id>9
+   AND m.hidden=0
+   AND (c.blocked=0 OR c.blocked=?)
+   AND ct.blocked=0
+   AND (m.txt LIKE ? OR ct.name LIKE ?)
+ ORDER BY m.timestamp DESC,m.id DESC;
+"#
         };
 
         self.sql
-            .query_map(
-                query,
-                params![chat_id, &strLikeInText, &strLikeBeg],
-                |row| row.get::<_, MsgId>("id"),
-                |rows| {
-                    let mut ret = Vec::new();
-                    for id in rows {
-                        ret.push(id?);
-                    }
-                    Ok(ret)
-                },
-            )
+            .query_values(query, paramsx![chat_id, strLikeInText, strLikeBeg])
+            .await
             .unwrap_or_default()
     }
 
-    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        folder_name.as_ref() == "INBOX"
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self.sql.get_raw_config(self, "configured_sentbox_folder");
-        if let Some(name) = sentbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+    pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self.sql.get_raw_config(self, "configured_mvbox_folder");
-
-        if let Some(name) = mvbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+    pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove) {
+    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
+        if !self.get_config_bool(Config::MvboxMove).await {
             return;
         }
 
-        if self.is_mvbox(folder) {
+        if self.is_mvbox(folder).await {
             return;
         }
-        if let Ok(msg) = Message::load_from_db(self, msg_id) {
+        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
             if msg.is_setupmessage() {
                 // do not move setup messages;
                 // there may be a non-delta device that wants to handle it
@@ -441,30 +456,32 @@ impl Context {
             match msg.is_dc_message {
                 MessengerMessage::No => {}
                 MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job_add(
+                    job::add(
                         self,
-                        Action::MoveMsg,
-                        msg.id.to_u32() as i32,
-                        Params::new(),
-                        0,
-                    );
+                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
+                    )
+                    .await;
                 }
             }
         }
     }
 }
 
-impl Drop for Context {
-    fn drop(&mut self) {
-        info!(self, "disconnecting inbox-thread",);
-        self.inbox_thread.read().unwrap().imap.disconnect(self);
-        info!(self, "disconnecting sentbox-thread",);
-        self.sentbox_thread.read().unwrap().imap.disconnect(self);
-        info!(self, "disconnecting mvbox-thread",);
-        self.mvbox_thread.read().unwrap().imap.disconnect(self);
-        info!(self, "disconnecting SMTP");
-        self.smtp.clone().lock().unwrap().disconnect();
-        self.sql.close(self);
+impl InnerContext {
+    async fn is_io_running(&self) -> bool {
+        self.scheduler.read().await.is_running()
+    }
+
+    async fn stop_io(&self) {
+        assert!(self.is_io_running().await, "context is already stopped");
+        let token = {
+            let lock = &*self.scheduler.read().await;
+            lock.pre_stop().await
+        };
+        {
+            let lock = &mut *self.scheduler.write().await;
+            lock.stop(token).await;
+        }
     }
 }
 
@@ -473,6 +490,7 @@ impl Default for RunningState {
         RunningState {
             ongoing_running: false,
             shall_stop_ongoing: true,
+            cancel_sender: None,
         }
     }
 }
@@ -484,28 +502,6 @@ pub(crate) struct BobStatus {
     pub qr_scan: Option<Lot>,
 }
 
-#[derive(Debug, PartialEq)]
-pub(crate) enum PerformJobsNeeded {
-    Not,
-    AtOnce,
-    AvoidDos,
-}
-
-impl Default for PerformJobsNeeded {
-    fn default() -> Self {
-        Self::Not
-    }
-}
-
-#[derive(Default, Debug)]
-pub struct SmtpState {
-    pub idle: bool,
-    pub suspended: bool,
-    pub doing_jobs: bool,
-    pub(crate) perform_jobs_needed: PerformJobsNeeded,
-    pub probe_network: bool,
-}
-
 pub fn get_version_str() -> &'static str {
     &DC_VERSION_STR
 }
@@ -516,81 +512,81 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[test]
-    fn test_wrong_db() {
+    #[async_std::test]
+    async fn test_wrong_db() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| ()), "FakeOs".into(), dbfile);
+        let res = Context::new("FakeOs".into(), dbfile.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn test_get_fresh_msgs() {
-        let t = dummy_context();
-        let fresh = t.ctx.get_fresh_msgs();
+    #[async_std::test]
+    async fn test_get_fresh_msgs() {
+        let t = TestContext::new().await;
+        let fresh = t.ctx.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
 
-    #[test]
-    fn test_blobdir_exists() {
+    #[async_std::test]
+    async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
+        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
 
-    #[test]
-    fn test_wrong_blogdir() {
+    #[async_std::test]
+    async fn test_wrong_blogdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile);
+        let res = Context::new("FakeOS".into(), dbfile.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn test_sqlite_parent_not_exists() {
+    #[async_std::test]
+    async fn test_sqlite_parent_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
+        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
 
-    #[test]
-    fn test_with_empty_blobdir() {
+    #[async_std::test]
+    async fn test_with_empty_blobdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn test_with_blobdir_not_exists() {
+    #[async_std::test]
+    async fn test_with_blobdir_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn no_crashes_on_context_deref() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn no_crashes_on_context_deref() {
+        let t = TestContext::new().await;
         std::mem::drop(t.ctx);
     }
 
-    #[test]
-    fn test_get_info() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_get_info() {
+        let t = TestContext::new().await;
 
-        let info = t.ctx.get_info();
+        let info = t.ctx.get_info().await;
         assert!(info.get("database_dir").is_some());
     }
 

src/dc_tools.rs

@@ -3,16 +3,17 @@
 
 use core::cmp::{max, min};
 use std::borrow::Cow;
-use std::path::{Path, PathBuf};
+use std::fmt;
 use std::str::FromStr;
-use std::time::SystemTime;
-use std::{fmt, fs};
+use std::time::{Duration, SystemTime};
 
+use async_std::path::{Path, PathBuf};
+use async_std::{fs, io};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::{bail, ensure, Error};
+use crate::error::{bail, Error};
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
@@ -75,6 +76,14 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String {
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
+pub fn duration_to_str(duration: Duration) -> String {
+    let secs = duration.as_secs();
+    let h = secs / 3600;
+    let m = (secs % 3600) / 60;
+    let s = (secs % 3600) % 60;
+    format!("{}h {}m {}s", h, m, s)
+}
+
 pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
@@ -99,9 +108,9 @@ const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 // returns the currently smeared timestamp,
 // may be used to check if call to dc_create_smeared_timestamp() is needed or not.
 // the returned timestamp MUST NOT be used to be sent out or saved in the database!
-pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
+pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     let mut now = time();
-    let ts = *context.last_smeared_timestamp.read().unwrap();
+    let ts = *context.last_smeared_timestamp.read().await;
     if ts >= now {
         now = ts + 1;
     }
@@ -110,11 +119,11 @@ pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
 }
 
 // returns a timestamp that is guaranteed to be unique.
-pub(crate) fn dc_create_smeared_timestamp(context: &Context) -> i64 {
+pub(crate) async fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
 
-    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
     if ret <= *last_smeared_timestamp {
         ret = *last_smeared_timestamp + 1;
         if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE {
@@ -129,12 +138,12 @@ pub(crate) fn dc_create_smeared_timestamp(context: &Context) -> i64 {
 // creates `count` timestamps that are guaranteed to be unique.
 // the frist created timestamps is returned directly,
 // get the other timestamps just by adding 1..count-1
-pub(crate) fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
+pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
     let now = time();
     let count = count as i64;
     let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count;
 
-    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
     start = max(*last_smeared_timestamp + 1, start);
 
     *last_smeared_timestamp = start + count - 1;
@@ -240,11 +249,8 @@ pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
 ///
 /// If `path` starts with "$BLOBDIR", replaces it with the blobdir path.
 /// Otherwise, returns path as is.
-pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
-    context: &Context,
-    path: P,
-) -> std::path::PathBuf {
-    let p: &std::path::Path = path.as_ref();
+pub(crate) fn dc_get_abs_path<P: AsRef<Path>>(context: &Context, path: P) -> PathBuf {
+    let p: &Path = path.as_ref();
     if let Ok(p) = p.strip_prefix("$BLOBDIR") {
         context.get_blobdir().join(p)
     } else {
@@ -252,20 +258,20 @@ pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) fn dc_get_filebytes(context: &Context, path: impl AsRef<std::path::Path>) -> u64 {
+pub(crate) async fn dc_get_filebytes(context: &Context, path: impl AsRef<Path>) -> u64 {
     let path_abs = dc_get_abs_path(context, &path);
-    match fs::metadata(&path_abs) {
+    match fs::metadata(&path_abs).await {
         Ok(meta) => meta.len() as u64,
         Err(_err) => 0,
     }
 }
 
-pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
+pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists() {
+    if !path_abs.exists().await {
         return false;
     }
-    if !path_abs.is_file() {
+    if !path_abs.is_file().await {
         warn!(
             context,
             "refusing to delete non-file \"{}\".",
@@ -275,9 +281,9 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
     }
 
     let dpath = format!("{}", path.as_ref().to_string_lossy());
-    match fs::remove_file(path_abs) {
+    match fs::remove_file(path_abs).await {
         Ok(_) => {
-            context.call_cb(Event::DeletedBlobFile(dpath));
+            context.emit_event(Event::DeletedBlobFile(dpath));
             true
         }
         Err(err) => {
@@ -287,13 +293,13 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
     }
 }
 
-pub(crate) fn dc_copy_file(
+pub(crate) async fn dc_copy_file(
     context: &Context,
-    src_path: impl AsRef<std::path::Path>,
-    dest_path: impl AsRef<std::path::Path>,
+    src_path: impl AsRef<Path>,
+    dest_path: impl AsRef<Path>,
 ) -> bool {
     let src_abs = dc_get_abs_path(context, &src_path);
-    let mut src_file = match fs::File::open(&src_abs) {
+    let mut src_file = match fs::File::open(&src_abs).await {
         Ok(file) => file,
         Err(err) => {
             warn!(
@@ -311,6 +317,7 @@ pub(crate) fn dc_copy_file(
         .create_new(true)
         .write(true)
         .open(&dest_abs)
+        .await
     {
         Ok(file) => file,
         Err(err) => {
@@ -324,7 +331,7 @@ pub(crate) fn dc_copy_file(
         }
     };
 
-    match std::io::copy(&mut src_file, &mut dest_file) {
+    match io::copy(&mut src_file, &mut dest_file).await {
         Ok(_) => true,
         Err(err) => {
             error!(
@@ -336,20 +343,20 @@ pub(crate) fn dc_copy_file(
             );
             {
                 // Attempt to remove the failed file, swallow errors resulting from that.
-                fs::remove_file(dest_abs).ok();
+                fs::remove_file(dest_abs).await.ok();
             }
             false
         }
     }
 }
 
-pub(crate) fn dc_create_folder(
+pub(crate) async fn dc_create_folder(
     context: &Context,
-    path: impl AsRef<std::path::Path>,
-) -> Result<(), std::io::Error> {
+    path: impl AsRef<Path>,
+) -> Result<(), io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists() {
-        match fs::create_dir_all(path_abs) {
+    if !path_abs.exists().await {
+        match fs::create_dir_all(path_abs).await {
             Ok(_) => Ok(()),
             Err(err) => {
                 warn!(
@@ -367,13 +374,13 @@ pub(crate) fn dc_create_folder(
 }
 
 /// Write a the given content to provied file path.
-pub(crate) fn dc_write_file(
+pub(crate) async fn dc_write_file(
     context: &Context,
     path: impl AsRef<Path>,
     buf: &[u8],
-) -> Result<(), std::io::Error> {
+) -> Result<(), io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    fs::write(&path_abs, buf).map_err(|err| {
+    fs::write(&path_abs, buf).await.map_err(|err| {
         warn!(
             context,
             "Cannot write {} bytes to \"{}\": {}",
@@ -385,13 +392,10 @@ pub(crate) fn dc_write_file(
     })
 }
 
-pub fn dc_read_file<P: AsRef<std::path::Path>>(
-    context: &Context,
-    path: P,
-) -> Result<Vec<u8>, Error> {
+pub async fn dc_read_file<P: AsRef<Path>>(context: &Context, path: P) -> Result<Vec<u8>, Error> {
     let path_abs = dc_get_abs_path(context, &path);
 
-    match fs::read(&path_abs) {
+    match fs::read(&path_abs).await {
         Ok(bytes) => Ok(bytes),
         Err(err) => {
             warn!(
@@ -405,13 +409,31 @@ pub fn dc_read_file<P: AsRef<std::path::Path>>(
     }
 }
 
-pub fn dc_open_file<P: AsRef<std::path::Path>>(
+pub async fn dc_open_file<P: AsRef<Path>>(context: &Context, path: P) -> Result<fs::File, Error> {
+    let path_abs = dc_get_abs_path(context, &path);
+
+    match fs::File::open(&path_abs).await {
+        Ok(bytes) => Ok(bytes),
+        Err(err) => {
+            warn!(
+                context,
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
+            );
+            Err(err.into())
+        }
+    }
+}
+
+pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     context: &Context,
     path: P,
 ) -> Result<std::fs::File, Error> {
-    let path_abs = dc_get_abs_path(context, &path);
+    let p: PathBuf = path.as_ref().into();
+    let path_abs = dc_get_abs_path(context, p);
 
-    match fs::File::open(&path_abs) {
+    match std::fs::File::open(&path_abs) {
         Ok(bytes) => Ok(bytes),
         Err(err) => {
             warn!(
@@ -425,7 +447,7 @@ pub fn dc_open_file<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) fn dc_get_next_backup_path(
+pub(crate) async fn dc_get_next_backup_path(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -438,7 +460,7 @@ pub(crate) fn dc_get_next_backup_path(
     for i in 0..64 {
         let mut path = folder.clone();
         path.push(format!("{}-{}.bak", stem, i));
-        if !path.exists() {
+        if !path.exists().await {
             return Ok(path);
         }
     }
@@ -452,6 +474,14 @@ pub(crate) fn time() -> i64 {
         .as_secs() as i64
 }
 
+/// An invalid email address was encountered
+#[derive(Debug, thiserror::Error)]
+#[error("Invalid email address: {message} ({addr})")]
+pub struct InvalidEmailError {
+    message: String,
+    addr: String,
+}
+
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -475,7 +505,7 @@ pub struct EmailAddress {
 }
 
 impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, Error> {
+    pub fn new(input: &str) -> Result<Self, InvalidEmailError> {
         input.parse::<EmailAddress>()
     }
 }
@@ -487,31 +517,53 @@ impl fmt::Display for EmailAddress {
 }
 
 impl FromStr for EmailAddress {
-    type Err = Error;
+    type Err = InvalidEmailError;
 
     /// Performs a dead-simple parse of an email address.
-    fn from_str(input: &str) -> Result<EmailAddress, Error> {
-        ensure!(!input.is_empty(), "empty string is not valid");
+    fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
+        let err = |msg: &str| {
+            Err(InvalidEmailError {
+                message: msg.to_string(),
+                addr: input.to_string(),
+            })
+        };
+        if input.is_empty() {
+            return err("empty string is not valid");
+        }
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
+        if input
+            .chars()
+            .any(|c| c.is_whitespace() || c == '<' || c == '>')
+        {
+            return err("Email must not contain whitespaces, '>' or '<'");
+        }
+
         match &parts[..] {
             [domain, local] => {
-                ensure!(
-                    !local.is_empty(),
-                    "empty string is not valid for local part"
-                );
-                ensure!(domain.len() > 3, "domain is too short");
-
+                if local.is_empty() {
+                    return err("empty string is not valid for local part");
+                }
+                if domain.len() <= 3 {
+                    return err("domain is too short");
+                }
                 let dot = domain.find('.');
-                ensure!(dot.is_some(), "invalid domain");
-                ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
-
+                match dot {
+                    None => {
+                        return err("invalid domain");
+                    }
+                    Some(dot_idx) => {
+                        if dot_idx >= domain.len() - 2 {
+                            return err("invalid domain");
+                        }
+                    }
+                }
                 Ok(EmailAddress {
                     local: (*local).to_string(),
                     domain: (*domain).to_string(),
                 })
             }
-            _ => bail!("missing '@' character"),
+            _ => err("missing '@' character"),
         }
     }
 }
@@ -712,31 +764,35 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_file_handling() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_file_handling() {
+        let t = TestContext::new().await;
         let context = &t.ctx;
-        let dc_file_exist = |ctx: &Context, fname: &str| {
-            ctx.get_blobdir()
-                .join(Path::new(fname).file_name().unwrap())
-                .exists()
-        };
-
-        assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje"));
-        if dc_file_exist(context, "$BLOBDIR/foobar")
-            || dc_file_exist(context, "$BLOBDIR/dada")
-            || dc_file_exist(context, "$BLOBDIR/foobar.dadada")
-            || dc_file_exist(context, "$BLOBDIR/foobar-folder")
-        {
-            dc_delete_file(context, "$BLOBDIR/foobar");
-            dc_delete_file(context, "$BLOBDIR/dada");
-            dc_delete_file(context, "$BLOBDIR/foobar.dadada");
-            dc_delete_file(context, "$BLOBDIR/foobar-folder");
+        macro_rules! dc_file_exist {
+            ($ctx:expr, $fname:expr) => {
+                $ctx.get_blobdir()
+                    .join(Path::new($fname).file_name().unwrap())
+                    .exists()
+            };
         }
-        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content").is_ok());
-        assert!(dc_file_exist(context, "$BLOBDIR/foobar",));
-        assert!(!dc_file_exist(context, "$BLOBDIR/foobarx"));
-        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar"), 7);
+
+        assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
+        if dc_file_exist!(context, "$BLOBDIR/foobar").await
+            || dc_file_exist!(context, "$BLOBDIR/dada").await
+            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
+            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
+        {
+            dc_delete_file(context, "$BLOBDIR/foobar").await;
+            dc_delete_file(context, "$BLOBDIR/dada").await;
+            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
+            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
+        }
+        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
+            .await
+            .is_ok());
+        assert!(dc_file_exist!(context, "$BLOBDIR/foobar").await);
+        assert!(!dc_file_exist!(context, "$BLOBDIR/foobarx").await);
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar").await, 7);
 
         let abs_path = context
             .get_blobdir()
@@ -744,31 +800,33 @@ mod tests {
             .to_string_lossy()
             .to_string();
 
-        assert!(dc_file_exist(context, &abs_path));
+        assert!(dc_file_exist!(context, &abs_path).await);
 
-        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
 
         // attempting to copy a second time should fail
-        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
 
-        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada",), 7);
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada").await, 7);
 
-        let buf = dc_read_file(context, "$BLOBDIR/dada").unwrap();
+        let buf = dc_read_file(context, "$BLOBDIR/dada").await.unwrap();
 
         assert_eq!(buf.len(), 7);
         assert_eq!(&buf, b"content");
 
-        assert!(dc_delete_file(context, "$BLOBDIR/foobar"));
-        assert!(dc_delete_file(context, "$BLOBDIR/dada"));
-        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder").is_ok());
-        assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
-        assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
+        assert!(dc_delete_file(context, "$BLOBDIR/foobar").await);
+        assert!(dc_delete_file(context, "$BLOBDIR/dada").await);
+        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder")
+            .await
+            .is_ok());
+        assert!(dc_file_exist!(context, "$BLOBDIR/foobar-folder").await);
+        assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder").await);
 
         let fn0 = "$BLOBDIR/data.data";
-        assert!(dc_write_file(context, &fn0, b"content").is_ok());
+        assert!(dc_write_file(context, &fn0, b"content").await.is_ok());
 
-        assert!(dc_delete_file(context, &fn0));
-        assert!(!dc_file_exist(context, &fn0));
+        assert!(dc_delete_file(context, &fn0).await);
+        assert!(!dc_file_exist!(context, &fn0).await);
     }
 
     #[test]
@@ -785,15 +843,15 @@ mod tests {
         assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
     }
 
-    #[test]
-    fn test_create_smeared_timestamp() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_smeared_timestamp() {
+        let t = TestContext::new().await;
         assert_ne!(
-            dc_create_smeared_timestamp(&t.ctx),
-            dc_create_smeared_timestamp(&t.ctx)
+            dc_create_smeared_timestamp(&t.ctx).await,
+            dc_create_smeared_timestamp(&t.ctx).await
         );
         assert!(
-            dc_create_smeared_timestamp(&t.ctx)
+            dc_create_smeared_timestamp(&t.ctx).await
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -801,17 +859,50 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_create_smeared_timestamps() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_smeared_timestamps() {
+        let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
-        let next = dc_smeared_time(&t.ctx);
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
+        let next = dc_smeared_time(&t.ctx).await;
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
-        let next = dc_smeared_time(&t.ctx);
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
+        let next = dc_smeared_time(&t.ctx).await;
         assert!((start + count - 1) < next);
     }
+
+    #[test]
+    fn test_duration_to_str() {
+        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
+        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
+        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
+        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
+        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
+        assert_eq!(
+            duration_to_str(Duration::from_secs(59 * 60 + 59)),
+            "0h 59m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(59 * 60 + 60)),
+            "1h 0m 0s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
+            "2h 59m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
+            "3h 0m 0s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
+            "3h 0m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
+            "3h 1m 0s"
+        );
+    }
 }

src/e2ee.rs

@@ -1,19 +1,17 @@
 //! End-to-end encryption support.
 
 use std::collections::HashSet;
-use std::convert::TryFrom;
 
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
-use crate::constants::KeyGenType;
 use crate::context::Context;
-use crate::dc_tools::EmailAddress;
 use crate::error::*;
-use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::key::{self, Key, KeyPairUse, SignedPublicKey};
+use crate::headerdef::HeaderDef;
+use crate::headerdef::HeaderDefMap;
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -27,18 +25,18 @@ pub struct EncryptHelper {
 }
 
 impl EncryptHelper {
-    pub fn new(context: &Context) -> Result<EncryptHelper> {
+    pub async fn new(context: &Context) -> Result<EncryptHelper> {
         let prefer_encrypt =
-            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled))
+            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await)
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr) {
+        let addr = match context.get_config(Config::ConfiguredAddr).await {
             None => {
                 bail!("addr not configured!");
             }
             Some(addr) => addr,
         };
 
-        let public_key = load_or_generate_self_public_key(context, &addr)?;
+        let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -88,46 +86,44 @@ impl EncryptHelper {
     }
 
     /// Tries to encrypt the passed in `mail`.
-    pub fn encrypt(
-        &mut self,
+    pub async fn encrypt(
+        self,
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: &[(Option<Peerstate>, &str)],
+        peerstates: Vec<(Option<Peerstate>, &str)>,
     ) -> Result<String> {
-        let mut keyring = Keyring::default();
+        let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
         for (peerstate, addr) in peerstates
-            .iter()
-            .filter_map(|(state, addr)| state.as_ref().map(|s| (s, addr)))
+            .into_iter()
+            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate.peek_key(min_verified).ok_or_else(|| {
+            let key = peerstate.take_key(min_verified).ok_or_else(|| {
                 format_err!("proper enc-key for {} missing, cannot encrypt", addr)
             })?;
-            keyring.add_ref(key);
+            keyring.add(key);
         }
-        let public_key = Key::from(self.public_key.clone());
-        keyring.add_ref(&public_key);
-        let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql)
-            .ok_or_else(|| format_err!("missing own private key"))?;
+        keyring.add(self.public_key.clone());
+        let sign_key = SignedSecretKey::load_self(context).await?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
-        let ctext = pgp::pk_encrypt(&raw_message, &keyring, Some(&sign_key))?;
+        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key)).await?;
 
         Ok(ctext)
     }
 }
 
-pub fn try_decrypt(
+pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
     let from = mail
         .headers
-        .get_header_value(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
+        .get_header(HeaderDef::From_)
+        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -136,92 +132,52 @@ pub fn try_decrypt(
     let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
 
     if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &context.sql, &from);
+        peerstate = Peerstate::from_addr(context, &from).await.ok();
 
         if let Some(ref mut peerstate) = peerstate {
             if let Some(ref header) = autocryptheader {
                 peerstate.apply_header(&header, message_time);
-                peerstate.save_to_db(&context.sql, false)?;
+                peerstate.save_to_db(&context.sql, false).await?;
             } else if message_time > peerstate.last_seen_autocrypt && !contains_report(mail) {
                 peerstate.degrade_encryption(message_time);
-                peerstate.save_to_db(&context.sql, false)?;
+                peerstate.save_to_db(&context.sql, false).await?;
             }
         } else if let Some(ref header) = autocryptheader {
-            let p = Peerstate::from_header(context, header, message_time);
-            p.save_to_db(&context.sql, true)?;
+            let p = Peerstate::from_header(header, message_time);
+            p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
     }
 
     /* possibly perform decryption */
-    let mut private_keyring = Keyring::default();
-    let mut public_keyring_for_validate = Keyring::default();
-    let mut out_mail = None;
+    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
+    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
-    let self_addr = context.get_config(Config::ConfiguredAddr);
 
-    if let Some(self_addr) = self_addr {
-        if private_keyring.load_self_private_for_decrypting(context, self_addr, &context.sql) {
-            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-                peerstate = Peerstate::from_addr(&context, &context.sql, &from);
-            }
-            if let Some(ref peerstate) = peerstate {
-                if peerstate.degrade_event.is_some() {
-                    handle_degrade_event(context, &peerstate)?;
-                }
-                if let Some(ref key) = peerstate.gossip_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-                if let Some(ref key) = peerstate.public_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-            }
-
-            out_mail = decrypt_if_autocrypt_message(
-                context,
-                mail,
-                &private_keyring,
-                &public_keyring_for_validate,
-                &mut signatures,
-            )?;
+    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
+        peerstate = Peerstate::from_addr(&context, &from).await.ok();
+    }
+    if let Some(peerstate) = peerstate {
+        if peerstate.degrade_event.is_some() {
+            handle_degrade_event(context, &peerstate).await?;
+        }
+        if let Some(key) = peerstate.gossip_key {
+            public_keyring_for_validate.add(key);
+        }
+        if let Some(key) = peerstate.public_key {
+            public_keyring_for_validate.add(key);
         }
     }
-    Ok((out_mail, signatures))
-}
 
-/// Load public key from database or generate a new one.
-///
-/// This will load a public key from the database, generating and
-/// storing a new one when one doesn't exist yet.  Care is taken to
-/// only generate one key per context even when multiple threads call
-/// this function concurrently.
-fn load_or_generate_self_public_key(
-    context: &Context,
-    self_addr: impl AsRef<str>,
-) -> Result<SignedPublicKey> {
-    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
-        return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key"));
-    }
-    let _guard = context.generating_key_mutex.lock().unwrap();
-
-    // Check again in case the key was generated while we were waiting for the lock.
-    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
-        return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key"));
-    }
-
-    let start = std::time::Instant::now();
-
-    let keygen_type =
-        KeyGenType::from_i32(context.get_config_int(Config::KeyGenType)).unwrap_or_default();
-    info!(context, "Generating keypair with type {}", keygen_type);
-    let keypair = pgp::create_keypair(EmailAddress::new(self_addr.as_ref())?, keygen_type)?;
-    key::store_self_keypair(context, &keypair, KeyPairUse::Default)?;
-    info!(
+    let out_mail = decrypt_if_autocrypt_message(
         context,
-        "Keypair generated in {:.3}s.",
-        start.elapsed().as_secs()
-    );
-    Ok(keypair.public)
+        mail,
+        private_keyring,
+        public_keyring_for_validate,
+        &mut signatures,
+    )
+    .await?;
+    Ok((out_mail, signatures))
 }
 
 /// Returns a reference to the encrypted payload and validates the autocrypt structure.
@@ -251,12 +207,12 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
     Ok(&mail.subparts[1])
 }
 
-fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
     mail: &ParsedMail<'a>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     //  The returned bool is true if we detected an Autocrypt-encrypted
     // message and successfully decrypted it. Decryption then modifies the
@@ -275,21 +231,20 @@ fn decrypt_if_autocrypt_message<'a>(
     info!(context, "Detected Autocrypt-mime message");
 
     decrypt_part(
-        context,
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
         ret_valid_signatures,
     )
+    .await
 }
 
 /// Returns Ok(None) if nothing encrypted was found.
-fn decrypt_part(
-    _context: &Context,
+async fn decrypt_part(
     mail: &ParsedMail<'_>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
@@ -298,11 +253,12 @@ fn decrypt_part(
         ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
         let plain = pgp::pk_decrypt(
-            &data,
-            &private_keyring,
-            &public_keyring_for_validate,
+            data,
+            private_keyring,
+            public_keyring_for_validate,
             Some(ret_valid_signatures),
-        )?;
+        )
+        .await?;
 
         ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
         return Ok(Some(plain));
@@ -345,14 +301,18 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 ///
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
-pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
-        format_err!(concat!(
-            "Failed to get self address, ",
-            "cannot ensure secret key if not configured."
-        ))
-    })?;
-    load_or_generate_self_public_key(context, &self_addr)?;
+// TODO, remove this once deltachat::key::Key no longer exists.
+pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
+    let self_addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await
+        .ok_or_else(|| {
+            format_err!(concat!(
+                "Failed to get self address, ",
+                "cannot ensure secret key if not configured."
+            ))
+        })?;
+    SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }
 
@@ -365,17 +325,17 @@ mod tests {
     mod ensure_secret_key_exists {
         use super::*;
 
-        #[test]
-        fn test_prexisting() {
-            let t = dummy_context();
-            let test_addr = configure_alice_keypair(&t.ctx);
-            assert_eq!(ensure_secret_key_exists(&t.ctx).unwrap(), test_addr);
+        #[async_std::test]
+        async fn test_prexisting() {
+            let t = TestContext::new().await;
+            let test_addr = t.configure_alice().await;
+            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
         }
 
-        #[test]
-        fn test_not_configured() {
-            let t = dummy_context();
-            assert!(ensure_secret_key_exists(&t.ctx).is_err());
+        #[async_std::test]
+        async fn test_not_configured() {
+            let t = TestContext::new().await;
+            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
         }
     }
 
@@ -403,47 +363,6 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
-    mod load_or_generate_self_public_key {
-        use super::*;
-
-        #[test]
-        fn test_existing() {
-            let t = dummy_context();
-            let addr = configure_alice_keypair(&t.ctx);
-            let key = load_or_generate_self_public_key(&t.ctx, addr);
-            assert!(key.is_ok());
-        }
-
-        #[test]
-        fn test_generate() {
-            let t = dummy_context();
-            let addr = "alice@example.org";
-            let key0 = load_or_generate_self_public_key(&t.ctx, addr);
-            assert!(key0.is_ok());
-            let key1 = load_or_generate_self_public_key(&t.ctx, addr);
-            assert!(key1.is_ok());
-            assert_eq!(key0.unwrap(), key1.unwrap());
-        }
-
-        #[test]
-        fn test_generate_concurrent() {
-            use std::sync::Arc;
-            use std::thread;
-
-            let t = dummy_context();
-            let ctx = Arc::new(t.ctx);
-            let ctx0 = Arc::clone(&ctx);
-            let thr0 =
-                thread::spawn(move || load_or_generate_self_public_key(&ctx0, "alice@example.org"));
-            let ctx1 = Arc::clone(&ctx);
-            let thr1 =
-                thread::spawn(move || load_or_generate_self_public_key(&ctx1, "alice@example.org"));
-            let res0 = thr0.join().unwrap();
-            let res1 = thr1.join().unwrap();
-            assert_eq!(res0.unwrap(), res1.unwrap());
-        }
-    }
-
     #[test]
     fn test_has_decrypted_pgp_armor() {
         let data = b" -----BEGIN PGP MESSAGE-----";

src/error.rs

@@ -2,13 +2,6 @@
 
 pub use anyhow::{bail, ensure, format_err, Error, Result};
 
-// #[fail(display = "Invalid Message ID.")]
-// InvalidMsgId,
-// #[fail(display = "Watch folder not found {:?}", _0)]
-// WatchFolderNotFound(String),
-// #[fail(display = "Not Configured")]
-// NotConfigured,
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -1,12 +1,65 @@
 //! # Events specification
 
-use std::path::PathBuf;
-
+use async_std::path::PathBuf;
+use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
 use crate::message::MsgId;
 
+#[derive(Debug)]
+pub struct Events {
+    receiver: Receiver<Event>,
+    sender: Sender<Event>,
+}
+
+impl Default for Events {
+    fn default() -> Self {
+        let (sender, receiver) = channel(1_000);
+
+        Self { receiver, sender }
+    }
+}
+
+impl Events {
+    pub fn emit(&self, event: Event) {
+        match self.sender.try_send(event) {
+            Ok(()) => {}
+            Err(TrySendError::Full(event)) => {
+                // when we are full, we pop remove the oldest event and push on the new one
+                let _ = self.receiver.try_recv();
+
+                // try again
+                self.emit(event);
+            }
+            Err(TrySendError::Disconnected(_)) => {
+                unreachable!("unable to emit event, channel disconnected");
+            }
+        }
+    }
+
+    /// Retrieve the event emitter.
+    pub fn get_emitter(&self) -> EventEmitter {
+        EventEmitter(self.receiver.clone())
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EventEmitter(Receiver<Event>);
+
+impl EventEmitter {
+    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
+    pub fn recv_sync(&self) -> Option<Event> {
+        async_std::task::block_on(self.recv())
+    }
+
+    /// Blocking async recv of an event. Return `None` if the `Sender` has been droped.
+    pub async fn recv(&self) -> Option<Event> {
+        // TODO: change once we can use async channels internally.
+        self.0.recv().await.ok()
+    }
+}
+
 impl Event {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {
@@ -201,22 +254,4 @@ pub enum Event {
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
-
-    /// This event is sent out to the inviter when a joiner successfully joined a group.
-    /// @param data1 (int) chat_id
-    /// @param data2 (int) contact_id
-    #[strum(props(id = "2062"))]
-    SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 },
-
-    /// This event is sent for each contact added to a chat.
-    /// @param data1 (int) chat_id
-    /// @param data2 (int) contact_id
-    #[strum(props(id = "2063"))]
-    MemberAdded { chat_id: ChatId, contact_id: u32 },
-
-    /// This event is sent for each contact removed from a chat.
-    /// @param data1 (int) chat_id
-    /// @param data2 (int) contact_id
-    #[strum(props(id = "2064"))]
-    MemberRemoved { chat_id: ChatId, contact_id: u32 },
 }

src/headerdef.rs

@@ -21,6 +21,7 @@ pub enum HeaderDef {
     References,
     InReplyTo,
     Precedence,
+    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -53,12 +54,16 @@ impl HeaderDef {
 
 pub trait HeaderDefMap {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
 }
 
 impl HeaderDefMap for [MailHeader<'_>] {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
         self.get_first_value(headerdef.get_headername())
     }
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
+        self.get_first_header(headerdef.get_headername())
+    }
 }
 
 #[cfg(test)]

src/imap/client.rs

@@ -1,28 +1,89 @@
+use std::ops::{Deref, DerefMut};
+
 use async_imap::{
     error::{Error as ImapError, Result as ImapResult},
     Client as ImapClient,
 };
-use async_native_tls::TlsStream;
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
+use crate::login_param::dc_build_tls;
+
+use super::session::SessionStream;
 
 #[derive(Debug)]
-pub(crate) enum Client {
-    Secure(ImapClient<TlsStream<TcpStream>>),
-    Insecure(ImapClient<TcpStream>),
+pub(crate) struct Client {
+    is_secure: bool,
+    inner: ImapClient<Box<dyn SessionStream>>,
+}
+
+impl Deref for Client {
+    type Target = ImapClient<Box<dyn SessionStream>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl DerefMut for Client {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
+    }
 }
 
 impl Client {
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> std::result::Result<Session, (ImapError, Self)> {
+        let Client { inner, is_secure } = self;
+        let session = inner
+            .login(username, password)
+            .await
+            .map_err(|(err, client)| {
+                (
+                    err,
+                    Client {
+                        is_secure,
+                        inner: client,
+                    },
+                )
+            })?;
+        Ok(Session { inner: session })
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> std::result::Result<Session, (ImapError, Self)> {
+        let Client { inner, is_secure } = self;
+        let session =
+            inner
+                .authenticate(auth_type, authenticator)
+                .await
+                .map_err(|(err, client)| {
+                    (
+                        err,
+                        Client {
+                            is_secure,
+                            inner: client,
+                        },
+                    )
+                })?;
+        Ok(Session { inner: session })
+    }
+
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        certificate_checks: CertificateChecks,
+        strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
-        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
+        let tls = dc_build_tls(strict_tls);
+        let tls_stream: Box<dyn SessionStream> =
+            Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
         if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
             client.debug = true;
@@ -33,11 +94,14 @@ impl Client {
             .await
             .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
-        Ok(Client::Secure(client))
+        Ok(Client {
+            is_secure: true,
+            inner: client,
+        })
     }
 
     pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
+        let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
         if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
@@ -48,57 +112,28 @@ impl Client {
             .await
             .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
-        Ok(Client::Insecure(client))
+        Ok(Client {
+            is_secure: false,
+            inner: client,
+        })
     }
 
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
-        match self {
-            Client::Insecure(client) => {
-                let tls = dc_build_tls(certificate_checks);
-                let client_sec = client.secure(domain, tls).await?;
+    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
+        if self.is_secure {
+            Ok(self)
+        } else {
+            let Client { mut inner, .. } = self;
+            let tls = dc_build_tls(strict_tls);
+            inner.run_command_and_check_ok("STARTTLS", None).await?;
 
-                Ok(Client::Secure(client_sec))
-            }
-            // Nothing to do
-            Client::Secure(_) => Ok(self),
-        }
-    }
+            let stream = inner.into_inner();
+            let ssl_stream = tls.connect(domain.as_ref(), stream).await?;
+            let boxed: Box<dyn SessionStream> = Box::new(ssl_stream);
 
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
-        self,
-        auth_type: S,
-        authenticator: &A,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
-        self,
-        username: U,
-        password: P,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
+            Ok(Client {
+                is_secure: true,
+                inner: ImapClient::new(boxed),
+            })
         }
     }
 }

src/imap/idle.rs

@@ -1,14 +1,10 @@
 use super::Imap;
 
-use async_imap::extensions::idle::{Handle as ImapIdleHandle, IdleResponse};
-use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
+use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
-use async_std::task;
-use std::sync::atomic::Ordering;
 use std::time::{Duration, SystemTime};
 
-use crate::context::Context;
+use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::select_folder;
 use super::session::Session;
@@ -33,255 +29,203 @@ pub enum Error {
     SetupHandleError(#[from] super::Error),
 }
 
-#[derive(Debug)]
-pub(crate) enum IdleHandle {
-    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
-    Insecure(ImapIdleHandle<TcpStream>),
-}
-
-impl Session {
-    pub fn idle(self) -> IdleHandle {
-        match self {
-            Session::Secure(i) => {
-                let h = i.idle();
-                IdleHandle::Secure(h)
-            }
-            Session::Insecure(i) => {
-                let h = i.idle();
-                IdleHandle::Insecure(h)
-            }
-        }
-    }
-}
-
 impl Imap {
     pub fn can_idle(&self) -> bool {
-        task::block_on(async move { self.config.read().await.can_idle })
+        self.config.can_idle
     }
 
-    pub fn idle(&self, context: &Context, watch_folder: Option<String>) -> Result<()> {
-        task::block_on(async move {
-            if !self.can_idle() {
-                return Err(Error::IdleAbilityMissing);
+    pub async fn idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> Result<InterruptInfo> {
+        use futures::future::FutureExt;
+
+        if !self.can_idle() {
+            return Err(Error::IdleAbilityMissing);
+        }
+        self.setup_handle_if_needed(context).await?;
+
+        self.select_folder(context, watch_folder.clone()).await?;
+
+        let session = self.session.take();
+        let timeout = Duration::from_secs(23 * 60);
+        let mut info = Default::default();
+
+        if let Some(session) = session {
+            let mut handle = session.idle();
+            if let Err(err) = handle.init().await {
+                return Err(Error::IdleProtocolFailed(err));
             }
 
-            self.setup_handle_if_needed(context).await?;
+            let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
 
-            self.select_folder(context, watch_folder.clone()).await?;
-
-            let session = self.session.lock().await.take();
-            let timeout = Duration::from_secs(23 * 60);
-            if let Some(session) = session {
-                match session.idle() {
-                    // BEWARE: If you change the Secure branch you
-                    // typically also need to change the Insecure branch.
-                    IdleHandle::Secure(mut handle) => {
-                        handle.init().await?;
-
-                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
-                        *self.interrupt.lock().await = Some(interrupt);
-
-                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
-                            // interrupt_idle has happened before we
-                            // provided self.interrupt
-                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
-                            std::mem::drop(idle_wait);
-                            info!(context, "Idle wait was skipped");
-                        } else {
-                            info!(context, "Idle entering wait-on-remote state");
-                            match idle_wait.await {
-                                Ok(IdleResponse::NewData(_)) => {
-                                    info!(context, "Idle has NewData");
-                                }
-                                // TODO: idle_wait does not distinguish manual interrupts
-                                // from Timeouts if we would know it's a Timeout we could bail
-                                // directly and reconnect .
-                                Ok(IdleResponse::Timeout) => {
-                                    info!(context, "Idle-wait timeout or interruption");
-                                }
-                                Ok(IdleResponse::ManualInterrupt) => {
-                                    info!(context, "Idle wait was interrupted");
-                                }
-                                Err(err) => {
-                                    warn!(context, "Idle wait errored: {:?}", err);
-                                }
-                            }
-                        }
-                        // if we can't properly terminate the idle
-                        // protocol let's break the connection.
-                        let res =
-                            async_std::future::timeout(Duration::from_secs(15), handle.done())
-                                .await
-                                .map_err(|err| {
-                                    self.trigger_reconnect();
-                                    Error::IdleTimeout(err)
-                                })?;
-
-                        match res {
-                            Ok(session) => {
-                                *self.session.lock().await = Some(Session::Secure(session));
-                            }
-                            Err(err) => {
-                                // if we cannot terminate IDLE it probably
-                                // means that we waited long (with idle_wait)
-                                // but the network went away/changed
-                                self.trigger_reconnect();
-                                return Err(Error::IdleProtocolFailed(err));
-                            }
-                        }
-                    }
-                    IdleHandle::Insecure(mut handle) => {
-                        handle.init().await?;
-
-                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
-                        *self.interrupt.lock().await = Some(interrupt);
-
-                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
-                            // interrupt_idle has happened before we
-                            // provided self.interrupt
-                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
-                            std::mem::drop(idle_wait);
-                            info!(context, "Idle wait was skipped");
-                        } else {
-                            info!(context, "Idle entering wait-on-remote state");
-                            match idle_wait.await {
-                                Ok(IdleResponse::NewData(_)) => {
-                                    info!(context, "Idle has NewData");
-                                }
-                                // TODO: idle_wait does not distinguish manual interrupts
-                                // from Timeouts if we would know it's a Timeout we could bail
-                                // directly and reconnect .
-                                Ok(IdleResponse::Timeout) => {
-                                    info!(context, "Idle-wait timeout or interruption");
-                                }
-                                Ok(IdleResponse::ManualInterrupt) => {
-                                    info!(context, "Idle wait was interrupted");
-                                }
-                                Err(err) => {
-                                    warn!(context, "Idle wait errored: {:?}", err);
-                                }
-                            }
-                        }
-                        // if we can't properly terminate the idle
-                        // protocol let's break the connection.
-                        let res =
-                            async_std::future::timeout(Duration::from_secs(15), handle.done())
-                                .await
-                                .map_err(|err| {
-                                    self.trigger_reconnect();
-                                    Error::IdleTimeout(err)
-                                })?;
-
-                        match res {
-                            Ok(session) => {
-                                *self.session.lock().await = Some(Session::Insecure(session));
-                            }
-                            Err(err) => {
-                                // if we cannot terminate IDLE it probably
-                                // means that we waited long (with idle_wait)
-                                // but the network went away/changed
-                                self.trigger_reconnect();
-                                return Err(Error::IdleProtocolFailed(err));
-                            }
-                        }
-                    }
-                }
+            enum Event {
+                IdleResponse(IdleResponse),
+                Interrupt(InterruptInfo),
             }
 
-            Ok(())
-        })
-    }
-
-    pub(crate) fn fake_idle(&self, context: &Context, watch_folder: Option<String>) {
-        // Idle using polling. This is also needed if we're not yet configured -
-        // in this case, we're waiting for a configure job (and an interrupt).
-        task::block_on(async move {
-            let fake_idle_start_time = SystemTime::now();
-
-            info!(context, "IMAP-fake-IDLEing...");
-
-            let interrupt = stop_token::StopSource::new();
-
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let interval = async_std::stream::interval(Duration::from_secs(60));
-            let mut interrupt_interval = interrupt.stop_token().stop_stream(interval);
-            *self.interrupt.lock().await = Some(interrupt);
-            if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+            if self.skip_next_idle_wait {
                 // interrupt_idle has happened before we
                 // provided self.interrupt
-                self.skip_next_idle_wait.store(false, Ordering::SeqCst);
-                info!(context, "fake-idle wait was skipped");
-            } else {
-                // loop until we are interrupted or if we fetched something
-                while let Some(_) = interrupt_interval.next().await {
-                    // try to connect with proper login params
-                    // (setup_handle_if_needed might not know about them if we
-                    // never successfully connected)
-                    if let Err(err) = self.connect_configured(context) {
-                        warn!(context, "fake_idle: could not connect: {}", err);
-                        continue;
-                    }
-                    if self.config.read().await.can_idle {
-                        // we only fake-idled because network was gone during IDLE, probably
-                        break;
-                    }
-                    info!(context, "fake_idle is connected");
-                    // we are connected, let's see if fetching messages results
-                    // in anything.  If so, we behave as if IDLE had data but
-                    // will have already fetched the messages so perform_*_fetch
-                    // will not find any new.
+                self.skip_next_idle_wait = false;
+                drop(idle_wait);
+                drop(interrupt);
 
-                    if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder).await {
-                            Ok(res) => {
-                                info!(context, "fetch_new_messages returned {:?}", res);
-                                if res {
-                                    break;
-                                }
-                            }
-                            Err(err) => {
-                                error!(context, "could not fetch from folder: {}", err);
-                                self.trigger_reconnect()
-                            }
-                        }
+                info!(context, "Idle wait was skipped");
+            } else {
+                info!(context, "Idle entering wait-on-remote state");
+                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
+                    self.idle_interrupt.recv().map(|probe_network| {
+                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+                    }),
+                );
+
+                match fut.await {
+                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
+                        info!(context, "Idle has NewData");
+                    }
+                    // TODO: idle_wait does not distinguish manual interrupts
+                    // from Timeouts if we would know it's a Timeout we could bail
+                    // directly and reconnect .
+                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                        info!(context, "Idle-wait timeout or interruption");
+                    }
+                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                        info!(context, "Idle wait was interrupted");
+                    }
+                    Ok(Event::Interrupt(i)) => {
+                        info = i;
+                        info!(context, "Idle wait was interrupted");
+                    }
+                    Err(err) => {
+                        warn!(context, "Idle wait errored: {:?}", err);
                     }
                 }
             }
-            self.interrupt.lock().await.take();
 
-            info!(
-                context,
-                "IMAP-fake-IDLE done after {:.4}s",
-                SystemTime::now()
-                    .duration_since(fake_idle_start_time)
-                    .unwrap_or_default()
-                    .as_millis() as f64
-                    / 1000.,
-            );
-        })
+            // if we can't properly terminate the idle
+            // protocol let's break the connection.
+            let res = handle
+                .done()
+                .timeout(Duration::from_secs(15))
+                .await
+                .map_err(|err| {
+                    self.trigger_reconnect();
+                    Error::IdleTimeout(err)
+                })?;
+
+            match res {
+                Ok(session) => {
+                    self.session = Some(Session { inner: session });
+                }
+                Err(err) => {
+                    // if we cannot terminate IDLE it probably
+                    // means that we waited long (with idle_wait)
+                    // but the network went away/changed
+                    self.trigger_reconnect();
+                    return Err(Error::IdleProtocolFailed(err));
+                }
+            }
+        }
+
+        Ok(info)
     }
 
-    pub fn interrupt_idle(&self, context: &Context) {
-        task::block_on(async move {
-            let mut interrupt: Option<stop_token::StopSource> = self.interrupt.lock().await.take();
-            if interrupt.is_none() {
-                // idle wait is not running, signal it needs to skip
-                self.skip_next_idle_wait.store(true, Ordering::SeqCst);
+    pub(crate) async fn fake_idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> InterruptInfo {
+        // Idle using polling. This is also needed if we're not yet configured -
+        // in this case, we're waiting for a configure job (and an interrupt).
 
-                // meanwhile idle-wait may have produced the StopSource
-                interrupt = self.interrupt.lock().await.take();
+        let fake_idle_start_time = SystemTime::now();
+        info!(context, "IMAP-fake-IDLEing...");
+
+        // Do not poll, just wait for an interrupt when no folder is passed in.
+        if watch_folder.is_none() {
+            return self.idle_interrupt.recv().await.unwrap_or_default();
+        }
+
+        let mut info: InterruptInfo = Default::default();
+        if self.skip_next_idle_wait {
+            // interrupt_idle has happened before we
+            // provided self.interrupt
+            self.skip_next_idle_wait = false;
+            info!(context, "fake-idle wait was skipped");
+        } else {
+            // check every minute if there are new messages
+            // TODO: grow sleep durations / make them more flexible
+            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+
+            enum Event {
+                Tick,
+                Interrupt(InterruptInfo),
             }
-            // let's manually drop the StopSource
-            if interrupt.is_some() {
-                // the imap thread provided us a stop token but might
-                // not have entered idle_wait yet, give it some time
-                // for that to happen. XXX handle this without extra wait
-                // https://github.com/deltachat/deltachat-core-rust/issues/925
-                std::thread::sleep(Duration::from_millis(200));
-                info!(context, "low-level: dropping stop-source to interrupt idle");
-                std::mem::drop(interrupt)
-            }
-        });
+            // loop until we are interrupted or if we fetched something
+            info =
+                loop {
+                    use futures::future::FutureExt;
+                    match interval
+                        .next()
+                        .map(|_| Event::Tick)
+                        .race(self.idle_interrupt.recv().map(|probe_network| {
+                            Event::Interrupt(probe_network.unwrap_or_default())
+                        }))
+                        .await
+                    {
+                        Event::Tick => {
+                            // try to connect with proper login params
+                            // (setup_handle_if_needed might not know about them if we
+                            // never successfully connected)
+                            if let Err(err) = self.connect_configured(context).await {
+                                warn!(context, "fake_idle: could not connect: {}", err);
+                                continue;
+                            }
+                            if self.config.can_idle {
+                                // we only fake-idled because network was gone during IDLE, probably
+                                break InterruptInfo::new(false, None);
+                            }
+                            info!(context, "fake_idle is connected");
+                            // we are connected, let's see if fetching messages results
+                            // in anything.  If so, we behave as if IDLE had data but
+                            // will have already fetched the messages so perform_*_fetch
+                            // will not find any new.
+
+                            if let Some(ref watch_folder) = watch_folder {
+                                match self.fetch_new_messages(context, watch_folder).await {
+                                    Ok(res) => {
+                                        info!(context, "fetch_new_messages returned {:?}", res);
+                                        if res {
+                                            break InterruptInfo::new(false, None);
+                                        }
+                                    }
+                                    Err(err) => {
+                                        error!(context, "could not fetch from folder: {}", err);
+                                        self.trigger_reconnect()
+                                    }
+                                }
+                            }
+                        }
+                        Event::Interrupt(info) => {
+                            // Interrupt
+                            break info;
+                        }
+                    }
+                };
+        }
+
+        info!(
+            context,
+            "IMAP-fake-IDLE done after {:.4}s",
+            SystemTime::now()
+                .duration_since(fake_idle_start_time)
+                .unwrap_or_default()
+                .as_millis() as f64
+                / 1000.,
+        );
+
+        info
     }
 }

src/imap/select_folder.rs

@@ -23,17 +23,44 @@ pub enum Error {
 }
 
 impl Imap {
+    /// Issues a CLOSE command to expunge selected folder.
+    ///
+    /// CLOSE is considerably faster than an EXPUNGE, see
+    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
+        if let Some(ref folder) = self.config.selected_folder {
+            info!(context, "Expunge messages in \"{}\".", folder);
+
+            if let Some(ref mut session) = self.session {
+                match session.close().await {
+                    Ok(_) => {
+                        info!(context, "close/expunge succeeded");
+                    }
+                    Err(err) => {
+                        self.trigger_reconnect();
+                        return Err(Error::CloseExpungeFailed(err));
+                    }
+                }
+            } else {
+                return Err(Error::NoSession);
+            }
+        }
+        self.config.selected_folder = None;
+        self.config.selected_folder_needs_expunge = false;
+
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
-        &self,
+        &mut self,
         context: &Context,
         folder: Option<S>,
     ) -> Result<()> {
-        if self.session.lock().await.is_none() {
-            let mut cfg = self.config.write().await;
-            cfg.selected_folder = None;
-            cfg.selected_folder_needs_expunge = false;
+        if self.session.is_none() {
+            self.config.selected_folder = None;
+            self.config.selected_folder_needs_expunge = false;
             self.trigger_reconnect();
             return Err(Error::NoSession);
         }
@@ -41,7 +68,7 @@ impl Imap {
         // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
         // if there is _no_ new folder, we continue as we might want to expunge below.
         if let Some(ref folder) = folder {
-            if let Some(ref selected_folder) = self.config.read().await.selected_folder {
+            if let Some(ref selected_folder) = self.config.selected_folder {
                 if folder.as_ref() == selected_folder {
                     return Ok(());
                 }
@@ -49,33 +76,14 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.read().await.selected_folder_needs_expunge };
+        let needs_expunge = { self.config.selected_folder_needs_expunge };
         if needs_expunge {
-            if let Some(ref folder) = self.config.read().await.selected_folder {
-                info!(context, "Expunge messages in \"{}\".", folder);
-
-                // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see
-                // https://tools.ietf.org/html/rfc3501#section-6.4.2
-                if let Some(ref mut session) = &mut *self.session.lock().await {
-                    match session.close().await {
-                        Ok(_) => {
-                            info!(context, "close/expunge succeeded");
-                        }
-                        Err(err) => {
-                            self.trigger_reconnect();
-                            return Err(Error::CloseExpungeFailed(err));
-                        }
-                    }
-                } else {
-                    return Err(Error::NoSession);
-                }
-            }
-            self.config.write().await.selected_folder_needs_expunge = false;
+            self.close_folder(context).await?;
         }
 
         // select new folder
         if let Some(ref folder) = folder {
-            if let Some(ref mut session) = &mut *self.session.lock().await {
+            if let Some(ref mut session) = &mut self.session {
                 let res = session.select(folder).await;
 
                 // https://tools.ietf.org/html/rfc3501#section-6.3.1
@@ -84,21 +92,20 @@ impl Imap {
 
                 match res {
                     Ok(mailbox) => {
-                        let mut config = self.config.write().await;
-                        config.selected_folder = Some(folder.as_ref().to_string());
-                        config.selected_mailbox = Some(mailbox);
+                        self.config.selected_folder = Some(folder.as_ref().to_string());
+                        self.config.selected_mailbox = Some(mailbox);
                         Ok(())
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
                         self.trigger_reconnect();
-                        self.config.write().await.selected_folder = None;
+                        self.config.selected_folder = None;
                         Err(Error::ConnectionLost)
                     }
                     Err(async_imap::error::Error::Validate(_)) => {
                         Err(Error::BadFolderName(folder.as_ref().to_string()))
                     }
                     Err(err) => {
-                        self.config.write().await.selected_folder = None;
+                        self.config.selected_folder = None;
                         self.trigger_reconnect();
                         Err(Error::Other(err.to_string()))
                     }

src/imap/session.rs

@@ -1,172 +1,40 @@
-use async_imap::{
-    error::Result as ImapResult,
-    types::{Capabilities, Fetch, Mailbox, Name},
-    Session as ImapSession,
-};
+use std::ops::{Deref, DerefMut};
+
+use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
 use async_std::net::TcpStream;
-use async_std::prelude::*;
 
 #[derive(Debug)]
-pub(crate) enum Session {
-    Secure(ImapSession<TlsStream<TcpStream>>),
-    Insecure(ImapSession<TcpStream>),
+pub(crate) struct Session {
+    pub(super) inner: ImapSession<Box<dyn SessionStream>>,
+}
+
+pub(crate) trait SessionStream:
+    async_std::io::Read + async_std::io::Write + Unpin + Send + Sync + std::fmt::Debug
+{
+}
+
+impl SessionStream for TlsStream<Box<dyn SessionStream>> {}
+impl SessionStream for TlsStream<TcpStream> {}
+impl SessionStream for TcpStream {}
+
+impl Deref for Session {
+    type Target = ImapSession<Box<dyn SessionStream>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl DerefMut for Session {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
+    }
 }
 
 impl Session {
-    pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
-        let res = match self {
-            Session::Secure(i) => i.capabilities().await?,
-            Session::Insecure(i) => i.capabilities().await?,
-        };
-
-        Ok(res)
-    }
-
-    pub async fn list(
-        &mut self,
-        reference_name: Option<&str>,
-        mailbox_pattern: Option<&str>,
-    ) -> ImapResult<Vec<Name>> {
-        let res = match self {
-            Session::Secure(i) => {
-                i.list(reference_name, mailbox_pattern)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.list(reference_name, mailbox_pattern)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-        Ok(res)
-    }
-
-    pub async fn create<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.create(mailbox_name).await?,
-            Session::Insecure(i) => i.create(mailbox_name).await?,
-        }
-        Ok(())
-    }
-
-    pub async fn subscribe<S: AsRef<str>>(&mut self, mailbox: S) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.subscribe(mailbox).await?,
-            Session::Insecure(i) => i.subscribe(mailbox).await?,
-        }
-        Ok(())
-    }
-
-    pub async fn close(&mut self) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.close().await?,
-            Session::Insecure(i) => i.close().await?,
-        }
-        Ok(())
-    }
-
-    pub async fn select<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<Mailbox> {
-        let mbox = match self {
-            Session::Secure(i) => i.select(mailbox_name).await?,
-            Session::Insecure(i) => i.select(mailbox_name).await?,
-        };
-
-        Ok(mbox)
-    }
-
-    pub async fn fetch<S1, S2>(&mut self, sequence_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
-    where
-        S1: AsRef<str>,
-        S2: AsRef<str>,
-    {
-        let res = match self {
-            Session::Secure(i) => {
-                i.fetch(sequence_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.fetch(sequence_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-        Ok(res)
-    }
-
-    pub async fn uid_fetch<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
-    where
-        S1: AsRef<str>,
-        S2: AsRef<str>,
-    {
-        let res = match self {
-            Session::Secure(i) => {
-                i.uid_fetch(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.uid_fetch(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-
-        Ok(res)
-    }
-
-    pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
-    where
-        S1: AsRef<str>,
-        S2: AsRef<str>,
-    {
-        let res = match self {
-            Session::Secure(i) => {
-                i.uid_store(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.uid_store(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-        Ok(res)
-    }
-
-    pub async fn uid_mv<S1: AsRef<str>, S2: AsRef<str>>(
-        &mut self,
-        uid_set: S1,
-        mailbox_name: S2,
-    ) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.uid_mv(uid_set, mailbox_name).await?,
-            Session::Insecure(i) => i.uid_mv(uid_set, mailbox_name).await?,
-        }
-        Ok(())
-    }
-
-    pub async fn uid_copy<S1: AsRef<str>, S2: AsRef<str>>(
-        &mut self,
-        uid_set: S1,
-        mailbox_name: S2,
-    ) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.uid_copy(uid_set, mailbox_name).await?,
-            Session::Insecure(i) => i.uid_copy(uid_set, mailbox_name).await?,
-        }
-
-        Ok(())
+    pub fn idle(self) -> async_imap::extensions::idle::Handle<Box<dyn SessionStream>> {
+        let Session { inner } = self;
+        inner.idle()
     }
 }

src/imex.rs

@@ -1,9 +1,10 @@
 //! # Import/export module
 
-use core::cmp::{max, min};
-use std::path::Path;
+use std::any::Any;
+use std::cmp::{max, min};
 
-use num_traits::FromPrimitive;
+use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -16,8 +17,7 @@ use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
-use crate::job::*;
-use crate::key::{self, Key};
+use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -53,13 +53,10 @@ pub enum ImexMode {
 }
 
 /// Import/export things.
-/// For this purpose, the function creates a job that is executed in the IMAP-thread then;
-/// this requires to call dc_perform_inbox_jobs() regularly.
 ///
 /// What to do is defined by the *what* parameter.
 ///
-/// While dc_imex() returns immediately, the started job may take a while,
-/// you can stop it using dc_stop_ongoing_process(). During execution of the job,
+/// During execution of the job,
 /// some events are sent out:
 ///
 /// - A number of #DC_EVENT_IMEX_PROGRESS events are sent and may be used to create
@@ -68,73 +65,84 @@ pub enum ImexMode {
 /// - For each file written on export, the function sends #DC_EVENT_IMEX_FILE_WRITTEN
 ///
 /// Only one import-/export-progress can run at the same time.
-/// To cancel an import-/export-progress, use dc_stop_ongoing_process().
-pub fn imex(context: &Context, what: ImexMode, param1: Option<impl AsRef<Path>>) {
-    let mut param = Params::new();
-    param.set_int(Param::Cmd, what as i32);
-    if let Some(param1) = param1 {
-        param.set(Param::Arg, param1.as_ref().to_string_lossy());
-    }
+/// To cancel an import-/export-progress, drop the future returned by this function.
+pub async fn imex(
+    context: &Context,
+    what: ImexMode,
+    param1: Option<impl AsRef<Path>>,
+) -> Result<()> {
+    use futures::future::FutureExt;
 
-    job_kill_action(context, Action::ImexImap);
-    job_add(context, Action::ImexImap, 0, param, 0);
+    let cancel = context.alloc_ongoing().await?;
+    let res = imex_inner(context, what, param1)
+        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
+        .await;
+
+    context.free_ongoing().await;
+
+    res
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
-    let dir_iter = std::fs::read_dir(dir_name)?;
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
-    let mut newest_backup_path: Option<std::path::PathBuf> = None;
-    for dirent in dir_iter {
+    let mut newest_backup_path: Option<PathBuf> = None;
+    while let Some(dirent) = dir_iter.next().await {
         if let Ok(dirent) = dirent {
             let path = dirent.path();
             let name = dirent.file_name();
             let name = name.to_string_lossy();
             if name.starts_with("delta-chat") && name.ends_with(".bak") {
                 let sql = Sql::new();
-                if sql.open(context, &path, true) {
-                    let curr_backup_time = sql
-                        .get_raw_config_int(context, "backup_time")
-                        .unwrap_or_default();
-                    if curr_backup_time > newest_backup_time {
-                        newest_backup_path = Some(path);
-                        newest_backup_time = curr_backup_time;
-                    }
-                    info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                    sql.close(&context);
+                sql.open(context, &path, true).await?;
+                let curr_backup_time = sql
+                    .get_raw_config_int("backup_time")
+                    .await
+                    .unwrap_or_default();
+                if curr_backup_time > newest_backup_time {
+                    newest_backup_path = Some(path);
+                    newest_backup_time = curr_backup_time;
                 }
+                info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                sql.close().await;
             }
         }
     }
+
     match newest_backup_path {
         Some(path) => Ok(path.to_string_lossy().into_owned()),
         None => bail!("no backup found in {}", dir_name.display()),
     }
 }
 
-pub fn initiate_key_transfer(context: &Context) -> Result<String> {
-    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
-    let res = do_initiate_key_transfer(context);
-    context.free_ongoing();
+pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
+    use futures::future::FutureExt;
+
+    let cancel = context.alloc_ongoing().await?;
+    let res = do_initiate_key_transfer(context)
+        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
+        .await;
+
+    context.free_ongoing().await;
     res
 }
 
-fn do_initiate_key_transfer(context: &Context) -> Result<String> {
+async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     let mut msg: Message;
     let setup_code = create_setup_code(context);
     /* this may require a keypair to be created. this may take a second ... */
-    ensure!(!context.shall_stop_ongoing(), "canceled");
-    let setup_file_content = render_setup_file(context, &setup_code)?;
+    let setup_file_content = render_setup_file(context, &setup_code).await?;
     /* encrypting may also take a while ... */
-    ensure!(!context.shall_stop_ongoing(), "canceled");
     let setup_file_blob = BlobObject::create(
         context,
         "autocrypt-setup-message.html",
         setup_file_content.as_bytes(),
-    )?;
+    )
+    .await?;
 
-    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF)?;
+    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF).await?;
     msg = Message::default();
     msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
@@ -147,12 +155,11 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
         ForcePlaintext::NoAutocryptHeader as i32,
     );
 
-    ensure!(!context.shall_stop_ongoing(), "canceled");
-    let msg_id = chat::send_msg(context, chat_id, &mut msg)?;
+    let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
-    while !context.shall_stop_ongoing() {
-        std::thread::sleep(std::time::Duration::from_secs(1));
-        if let Ok(msg) = Message::load_from_db(context, msg_id) {
+    while !context.shall_stop_ongoing().await {
+        async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+        if let Ok(msg) = Message::load_from_db(context, msg_id).await {
             if msg.is_sent() {
                 info!(context, "... setup message sent.",);
                 break;
@@ -170,20 +177,18 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 /// Renders HTML body of a setup file message.
 ///
 /// The `passphrase` must be at least 2 characters long.
-pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
+pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
     ensure!(
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let self_addr = e2ee::ensure_secret_key_exists(context)?;
-    let private_key = Key::from_self_private(context, self_addr, &context.sql)
-        .ok_or_else(|| format_err!("Failed to get private key."))?;
-    let ac_headers = match context.get_config_bool(Config::E2eeEnabled) {
+    let private_key = SignedSecretKey::load_self(context).await?;
+    let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
+    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -195,8 +200,8 @@ pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String>
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
-    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject);
-    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody);
+    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject).await;
+    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody).await;
     let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
     Ok(format!(
         concat!(
@@ -239,8 +244,8 @@ pub fn create_setup_code(_context: &Context) -> String {
     ret
 }
 
-fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
-    if !context.sql.get_raw_config_bool(context, "bcc_self") {
+async fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
+    if !context.sql.get_raw_config_bool("bcc_self").await {
         let mut msg = Message::new(Viewtype::Text);
         // TODO: define this as a stockstring once the wording is settled.
         msg.text = Some(
@@ -249,26 +254,30 @@ fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
              go to the settings and enable \"Send copy to self\"."
                 .to_string(),
         );
-        chat::add_device_msg(context, Some("bcc-self-hint"), Some(&mut msg))?;
+        chat::add_device_msg(context, Some("bcc-self-hint"), Some(&mut msg)).await?;
     }
     Ok(())
 }
 
-pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str) -> Result<()> {
+pub async fn continue_key_transfer(
+    context: &Context,
+    msg_id: MsgId,
+    setup_code: &str,
+) -> Result<()> {
     ensure!(!msg_id.is_special(), "wrong id");
 
-    let msg = Message::load_from_db(context, msg_id)?;
+    let msg = Message::load_from_db(context, msg_id).await?;
     ensure!(
         msg.is_setupmessage(),
         "Message is no Autocrypt Setup Message."
     );
 
     if let Some(filename) = msg.get_file(context) {
-        let file = dc_open_file(context, filename)?;
+        let file = dc_open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
-        let armored_key = decrypt_setup_file(context, &sc, file)?;
-        set_self_key(context, &armored_key, true, true)?;
-        maybe_add_bcc_self_device_msg(context)?;
+        let armored_key = decrypt_setup_file(&sc, file).await?;
+        set_self_key(context, &armored_key, true, true).await?;
+        maybe_add_bcc_self_device_msg(context).await?;
 
         Ok(())
     } else {
@@ -276,19 +285,15 @@ pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str)
     }
 }
 
-fn set_self_key(
+async fn set_self_key(
     context: &Context,
     armored: &str,
     set_default: bool,
     prefer_encrypt_required: bool,
 ) -> Result<()> {
     // try hard to only modify key-state
-    let keys = Key::from_armored_string(armored, KeyType::Private)
-        .and_then(|(k, h)| if k.verify() { Some((k, h)) } else { None })
-        .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
-
-    ensure!(keys.is_some(), "Not a valid private key");
-    let (private_key, public_key, header) = keys.unwrap();
+    let (private_key, header) = SignedSecretKey::from_asc(armored)?;
+    let public_key = private_key.split_public_key()?;
     let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
     match preferencrypt.map(|s| s.as_str()) {
         Some(headerval) => {
@@ -301,7 +306,8 @@ fn set_self_key(
             };
             context
                 .sql
-                .set_raw_config_int(context, "e2ee_enabled", e2ee_enabled)?;
+                .set_raw_config_int(context, "e2ee_enabled", e2ee_enabled)
+                .await?;
         }
         None => {
             if prefer_encrypt_required {
@@ -310,18 +316,13 @@ fn set_self_key(
         }
     };
 
-    let self_addr = context.get_config(Config::ConfiguredAddr);
+    let self_addr = context.get_config(Config::ConfiguredAddr).await;
     ensure!(self_addr.is_some(), "Missing self addr");
     let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
-
-    let (public, secret) = match (public_key, private_key) {
-        (Key::Public(p), Key::Secret(s)) => (p, s),
-        _ => bail!("wrong keys unpacked"),
-    };
     let keypair = pgp::KeyPair {
         addr,
-        public,
-        secret,
+        public: public_key,
+        secret: private_key,
     };
     key::store_self_keypair(
         context,
@@ -331,16 +332,16 @@ fn set_self_key(
         } else {
             key::KeyPairUse::ReadOnly
         },
-    )?;
+    )
+    .await?;
     Ok(())
 }
 
-fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
-    _context: &Context,
+async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     file: T,
 ) -> Result<String> {
-    let plain_bytes = pgp::symm_decrypt(passphrase, file)?;
+    let plain_bytes = pgp::symm_decrypt(passphrase, file).await?;
     let plain_text = std::string::String::from_utf8(plain_bytes)?;
 
     Ok(plain_text)
@@ -359,52 +360,51 @@ pub fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-#[allow(non_snake_case)]
-pub fn JobImexImap(context: &Context, job: &Job) -> Result<()> {
-    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
-    let what: Option<ImexMode> = job.param.get_int(Param::Cmd).and_then(ImexMode::from_i32);
-    let param = job.param.get(Param::Arg).unwrap_or_default();
+async fn imex_inner(
+    context: &Context,
+    what: ImexMode,
+    param: Option<impl AsRef<Path>>,
+) -> Result<()> {
+    ensure!(param.is_some(), "No Import/export dir/file given.");
 
-    ensure!(!param.is_empty(), "No Import/export dir/file given.");
     info!(context, "Import/export process started.");
-    context.call_cb(Event::ImexProgress(10));
+    context.emit_event(Event::ImexProgress(10));
 
-    ensure!(context.sql.is_open(), "Database not opened.");
-    if what == Some(ImexMode::ExportBackup) || what == Some(ImexMode::ExportSelfKeys) {
+    ensure!(context.sql.is_open().await, "Database not opened.");
+
+    let path = param.unwrap();
+    if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
-        if e2ee::ensure_secret_key_exists(context).is_err() {
-            context.free_ongoing();
+        if e2ee::ensure_secret_key_exists(context).await.is_err() {
             bail!("Cannot create private key or private key not available.");
         } else {
-            dc_create_folder(context, &param)?;
+            dc_create_folder(context, &path).await?;
         }
     }
-    let path = Path::new(param);
+
     let success = match what {
-        Some(ImexMode::ExportSelfKeys) => export_self_keys(context, path),
-        Some(ImexMode::ImportSelfKeys) => import_self_keys(context, path),
-        Some(ImexMode::ExportBackup) => export_backup(context, path),
-        Some(ImexMode::ImportBackup) => import_backup(context, path),
-        None => {
-            bail!("unknown IMEX type");
-        }
+        ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
+        ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
+        ImexMode::ExportBackup => export_backup(context, path).await,
+        ImexMode::ImportBackup => import_backup(context, path).await,
     };
-    context.free_ongoing();
+
     match success {
         Ok(()) => {
             info!(context, "IMEX successfully completed");
-            context.call_cb(Event::ImexProgress(1000));
+            context.emit_event(Event::ImexProgress(1000));
             Ok(())
         }
         Err(err) => {
-            context.call_cb(Event::ImexProgress(0));
+            error!(context, "IMEX FAILED: {}", err);
+            context.emit_event(Event::ImexProgress(0));
             bail!("IMEX FAILED to complete: {}", err);
         }
     }
 }
 
 /// Import Backup
-fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -413,85 +413,81 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     );
 
     ensure!(
-        !context.is_configured(),
+        !context.is_configured().await,
         "Cannot import backups to accounts in use."
     );
-    context.sql.close(&context);
-    dc_delete_file(context, context.get_dbfile());
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
     ensure!(
-        !context.get_dbfile().exists(),
+        !context.get_dbfile().exists().await,
         "Cannot delete old database."
     );
 
     ensure!(
-        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()),
+        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()).await,
         "could not copy file"
     );
     /* error already logged */
     /* re-open copied database file */
-    ensure!(
-        context.sql.open(&context, &context.get_dbfile(), false),
-        "could not re-open db"
-    );
-
-    delete_and_reset_all_device_msgs(&context)?;
-
-    let total_files_cnt = context
+    context
         .sql
-        .query_get_value::<_, isize>(context, "SELECT COUNT(*) FROM backup_blobs;", params![])
-        .unwrap_or_default() as usize;
+        .open(&context, &context.get_dbfile(), false)
+        .await?;
+
+    delete_and_reset_all_device_msgs(&context).await?;
+
+    let total_files_cnt: i32 = context
+        .sql
+        .query_value("SELECT COUNT(*) FROM backup_blobs;", paramsx![])
+        .await
+        .unwrap_or_default();
+    let total_files_cnt = total_files_cnt as usize;
     info!(
         context,
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
     );
 
-    let res = context.sql.query_map(
-        "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
-        params![],
-        |row| {
-            let name: String = row.get(0)?;
-            let blob: Vec<u8> = row.get(1)?;
+    let pool = context.sql.get_pool().await?;
+    let mut files = sqlx::query_as("SELECT file_name, file_content FROM backup_blobs ORDER BY id;")
+        .fetch(&pool);
 
-            Ok((name, blob))
-        },
-        |files| {
-            for (processed_files_cnt, file) in files.enumerate() {
-                let (file_name, file_blob) = file?;
-                if context.shall_stop_ongoing() {
-                    return Ok(false);
-                }
-                let mut permille = processed_files_cnt * 1000 / total_files_cnt;
-                if permille < 10 {
-                    permille = 10
-                }
-                if permille > 990 {
-                    permille = 990
-                }
-                context.call_cb(Event::ImexProgress(permille));
-                if file_blob.is_empty() {
-                    continue;
-                }
+    let mut processed_files_cnt = 0;
+    while let Some(files_result) = files.next().await {
+        let (file_name, file_blob): (String, Vec<u8>) = files_result?;
+        if context.shall_stop_ongoing().await {
+            break;
+        }
 
-                let path_filename = context.get_blobdir().join(file_name);
-                dc_write_file(context, &path_filename, &file_blob)?;
-            }
-            Ok(true)
-        },
+        let mut permille = processed_files_cnt * 1000 / total_files_cnt;
+        if permille < 10 {
+            permille = 10
+        }
+        if permille > 990 {
+            permille = 990
+        }
+        context.emit_event(Event::ImexProgress(permille));
+        if file_blob.is_empty() {
+            continue;
+        }
+
+        let path_filename = context.get_blobdir().join(file_name);
+        dc_write_file(context, &path_filename, &file_blob).await?;
+
+        processed_files_cnt += 1;
+    }
+
+    ensure!(
+        processed_files_cnt == total_files_cnt,
+        "received stop signal"
     );
 
-    match res {
-        Ok(all_files_extracted) => {
-            if all_files_extracted {
-                // only delete backup_blobs if all files were successfully extracted
-                sql::execute(context, &context.sql, "DROP TABLE backup_blobs;", params![])?;
-                sql::try_execute(context, &context.sql, "VACUUM;").ok();
-                Ok(())
-            } else {
-                bail!("received stop signal");
-            }
-        }
-        Err(err) => Err(err.into()),
-    }
+    context
+        .sql
+        .execute("DROP TABLE backup_blobs;", paramsx![])
+        .await?;
+    context.sql.execute("VACUUM;", paramsx![]).await?;
+
+    Ok(())
 }
 
 /*******************************************************************************
@@ -499,28 +495,31 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
  ******************************************************************************/
 /* the FILE_PROGRESS macro calls the callback with the permille of files processed.
 The macro avoids weird values of 0% or 100% while still working. */
-fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let dest_path_filename = dc_get_next_backup_path(dir, now)?;
+    let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
     let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
-    sql::housekeeping(context);
+    sql::housekeeping(context).await?;
 
-    sql::try_execute(context, &context.sql, "VACUUM;").ok();
+    context.sql.execute("VACUUM;", paramsx![]).await.ok();
 
     // we close the database during the copy of the dbfile
-    context.sql.close(context);
+    context.sql.close().await;
     info!(
         context,
         "Backup '{}' to '{}'.",
         context.get_dbfile().display(),
         dest_path_filename.display(),
     );
-    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
-    context.sql.open(&context, &context.get_dbfile(), false);
+    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename).await;
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await?;
 
     if !copied {
         bail!(
@@ -530,87 +529,85 @@ fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         );
     }
     let dest_sql = Sql::new();
-    ensure!(
-        dest_sql.open(context, &dest_path_filename, false),
-        "could not open exported database {}",
-        dest_path_string
-    );
-    let res = match add_files_to_export(context, &dest_sql) {
+    dest_sql.open(context, &dest_path_filename, false).await?;
+
+    let res = match add_files_to_export(context, &dest_sql).await {
+        Ok(()) => {
+            dest_sql
+                .set_raw_config_int(context, "backup_time", now as i32)
+                .await?;
+            context.emit_event(Event::ImexFileWritten(dest_path_filename));
+            Ok(())
+        }
         Err(err) => {
-            dc_delete_file(context, &dest_path_filename);
+            dc_delete_file(context, &dest_path_filename).await;
             error!(context, "backup failed: {}", err);
             Err(err)
         }
-        Ok(()) => {
-            dest_sql.set_raw_config_int(context, "backup_time", now as i32)?;
-            context.call_cb(Event::ImexFileWritten(dest_path_filename));
-            Ok(())
-        }
     };
-    dest_sql.close(context);
+    dest_sql.close().await;
 
     Ok(res?)
 }
 
-fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
+async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
     // add all files as blobs to the database copy (this does not require
     // the source to be locked, neigher the destination as it is used only here)
-    if !sql.table_exists("backup_blobs") {
-        sql::execute(
-            context,
-            &sql,
+    if !sql.table_exists("backup_blobs").await? {
+        sql.execute(
             "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-            params![],
-        )?
+            paramsx![],
+        )
+        .await?;
     }
     // copy all files from BLOBDIR into backup-db
     let mut total_files_cnt = 0;
     let dir = context.get_blobdir();
-    let dir_handle = std::fs::read_dir(&dir)?;
-    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
+    let dir_handle = async_std::fs::read_dir(&dir).await?;
+    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count().await;
 
     info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-    // scan directory, pass 2: copy files
-    let dir_handle = std::fs::read_dir(&dir)?;
-    let exported_all_files = sql.prepare(
-        "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
-        |mut stmt, _| {
-            let mut processed_files_cnt = 0;
-            for entry in dir_handle {
-                let entry = entry?;
-                if context.shall_stop_ongoing() {
-                    return Ok(false);
-                }
-                processed_files_cnt += 1;
-                let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-                context.call_cb(Event::ImexProgress(permille));
 
-                let name_f = entry.file_name();
-                let name = name_f.to_string_lossy();
-                if name.starts_with("delta-chat") && name.ends_with(".bak") {
-                    continue;
-                }
-                info!(context, "EXPORT: copying filename={}", name);
-                let curr_path_filename = context.get_blobdir().join(entry.file_name());
-                if let Ok(buf) = dc_read_file(context, &curr_path_filename) {
-                    if buf.is_empty() {
-                        continue;
-                    }
-                    // bail out if we can't insert
-                    stmt.execute(params![name, buf])?;
-                }
+    // scan directory, pass 2: copy files
+    let mut dir_handle = async_std::fs::read_dir(&dir).await?;
+
+    let mut processed_files_cnt = 0;
+    while let Some(entry) = dir_handle.next().await {
+        let entry = entry?;
+        if context.shall_stop_ongoing().await {
+            return Ok(());
+        }
+        processed_files_cnt += 1;
+        let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
+        context.emit_event(Event::ImexProgress(permille));
+
+        let name_f = entry.file_name();
+        let name = name_f.to_string_lossy();
+        if name.starts_with("delta-chat") && name.ends_with(".bak") {
+            continue;
+        }
+        info!(context, "EXPORT: copying filename={}", name);
+        let curr_path_filename = context.get_blobdir().join(entry.file_name());
+        if let Ok(buf) = dc_read_file(context, &curr_path_filename).await {
+            if buf.is_empty() {
+                continue;
             }
-            Ok(true)
-        },
-    )?;
-    ensure!(exported_all_files, "canceled during export-files");
+            // bail out if we can't insert
+            sql.execute(
+                "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
+                paramsx![name.as_ref(), buf],
+            )
+            .await?;
+        }
+    }
+
     Ok(())
 }
 
 /*******************************************************************************
  * Classic key import
  ******************************************************************************/
-fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
     plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
     Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
@@ -621,8 +618,8 @@ fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     let mut imported_cnt = 0;
 
     let dir_name = dir.as_ref().to_string_lossy();
-    let dir_handle = std::fs::read_dir(&dir)?;
-    for entry in dir_handle {
+    let mut dir_handle = async_std::fs::read_dir(&dir).await?;
+    while let Some(entry) = dir_handle.next().await {
         let entry_fn = entry?.file_name();
         let name_f = entry_fn.to_string_lossy();
         let path_plus_name = dir.as_ref().join(&entry_fn);
@@ -642,10 +639,10 @@ fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
                 continue;
             }
         }
-        match dc_read_file(context, &path_plus_name) {
+        match dc_read_file(context, &path_plus_name).await {
             Ok(buf) => {
                 let armored = std::string::String::from_utf8_lossy(&buf);
-                if let Err(err) = set_self_key(context, &armored, set_default, false) {
+                if let Err(err) = set_self_key(context, &armored, set_default, false).await {
                     error!(context, "set_self_key: {}", err);
                     continue;
                 }
@@ -662,73 +659,82 @@ fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     Ok(())
 }
 
-fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     let mut export_errors = 0;
 
-    context.sql.query_map(
-        "SELECT id, public_key, private_key, is_default FROM keypairs;",
-        params![],
-        |row| {
-            let id = row.get(0)?;
-            let public_key_blob: Vec<u8> = row.get(1)?;
-            let public_key = Key::from_slice(&public_key_blob, KeyType::Public);
-            let private_key_blob: Vec<u8> = row.get(2)?;
-            let private_key = Key::from_slice(&private_key_blob, KeyType::Private);
-            let is_default: i32 = row.get(3)?;
+    let pool = context.sql.get_pool().await?;
 
-            Ok((id, public_key, private_key, is_default))
-        },
-        |keys| {
-            for key_pair in keys {
-                let (id, public_key, private_key, is_default) = key_pair?;
-                let id = Some(id).filter(|_| is_default != 0);
-                if let Some(key) = public_key {
-                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
-                        export_errors += 1;
-                    }
-                } else {
-                    export_errors += 1;
-                }
-                if let Some(key) = private_key {
-                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
-                        export_errors += 1;
-                    }
-                } else {
-                    export_errors += 1;
-                }
+    let mut keys = sqlx::query_as("SELECT id, public_key, private_key, is_default FROM keypairs;")
+        .fetch(&pool);
+
+    while let Some(keys_result) = keys.next().await {
+        let (id, public_key_blob, private_key_blob, is_default): (i64, Vec<u8>, Vec<u8>, i32) =
+            keys_result?;
+        let public_key = SignedPublicKey::from_slice(&public_key_blob);
+        let private_key = SignedSecretKey::from_slice(&private_key_blob);
+
+        let id = Some(id).filter(|_| is_default != 0);
+
+        if let Ok(key) = public_key {
+            if export_key_to_asc_file(context, &dir, id, &key)
+                .await
+                .is_err()
+            {
+                export_errors += 1;
             }
-
-            Ok(())
-        },
-    )?;
+        } else {
+            export_errors += 1;
+        }
+        if let Ok(key) = private_key {
+            if export_key_to_asc_file(context, &dir, id, &key)
+                .await
+                .is_err()
+            {
+                export_errors += 1;
+            }
+        } else {
+            export_errors += 1;
+        }
+    }
 
     ensure!(export_errors == 0, "errors while exporting keys");
+
     Ok(())
 }
 
 /*******************************************************************************
  * Classic key export
  ******************************************************************************/
-fn export_key_to_asc_file(
+async fn export_key_to_asc_file<T>(
     context: &Context,
     dir: impl AsRef<Path>,
     id: Option<i64>,
-    key: &Key,
-) -> std::io::Result<()> {
+    key: &T,
+) -> std::io::Result<()>
+where
+    T: DcKey + Any,
+{
     let file_name = {
-        let kind = if key.is_public() { "public" } else { "private" };
+        let any_key = key as &dyn Any;
+        let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "public"
+        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "private"
+        } else {
+            "unknown"
+        };
         let id = id.map_or("default".into(), |i| i.to_string());
-
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
     info!(context, "Exporting key {}", file_name.display());
-    dc_delete_file(context, &file_name);
+    dc_delete_file(context, &file_name).await;
 
-    let res = key.write_asc_to_file(&file_name, context);
+    let content = key.to_asc(None).into_bytes();
+    let res = dc_write_file(context, &file_name, &content).await;
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
-        context.call_cb(Event::ImexFileWritten(file_name));
+        context.emit_event(Event::ImexFileWritten(file_name));
     }
     res
 }
@@ -740,12 +746,12 @@ mod tests {
     use crate::test_utils::*;
     use ::pgp::armor::BlockType;
 
-    #[test]
-    fn test_render_setup_file() {
-        let t = test_context(Some(Box::new(logging_cb)));
+    #[async_std::test]
+    async fn test_render_setup_file() {
+        let t = TestContext::new().await;
 
-        configure_alice_keypair(&t.ctx);
-        let msg = render_setup_file(&t.ctx, "hello").unwrap();
+        t.configure_alice().await;
+        let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
         // In particular note the mixing of `\r\n` and `\n` depending
@@ -759,21 +765,22 @@ mod tests {
         assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 
-    #[test]
-    fn test_render_setup_file_newline_replace() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_render_setup_file_newline_replace() {
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+            .await
             .unwrap();
-        configure_alice_keypair(&t.ctx);
-        let msg = render_setup_file(&t.ctx, "pw").unwrap();
+        t.configure_alice().await;
+        let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
     }
 
-    #[test]
-    fn test_create_setup_code() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_setup_code() {
+        let t = TestContext::new().await;
         let setupcode = create_setup_code(&t.ctx);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
@@ -786,15 +793,17 @@ mod tests {
         assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
     }
 
-    #[test]
-    fn test_export_key_to_asc_file() {
-        let context = dummy_context();
-        let key = Key::from(alice_keypair().public);
+    #[async_std::test]
+    async fn test_export_key_to_asc_file() {
+        let context = TestContext::new().await;
+        let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
-        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key).is_ok());
+        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
+            .await
+            .is_ok());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{}/public-key-default.asc", blobdir);
-        let bytes = std::fs::read(&filename).unwrap();
+        let bytes = async_std::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
@@ -814,11 +823,8 @@ mod tests {
     const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
     const S_EM_SETUPFILE: &str = include_str!("../test-data/message/stress.txt");
 
-    #[test]
-    fn test_split_and_decrypt() {
-        let ctx = dummy_context();
-        let context = &ctx.ctx;
-
+    #[async_std::test]
+    async fn test_split_and_decrypt() {
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
         let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
         assert_eq!(typ, BlockType::Message);
@@ -828,12 +834,10 @@ mod tests {
         assert!(!base64.is_empty());
 
         let setup_file = S_EM_SETUPFILE.to_string();
-        let decrypted = decrypt_setup_file(
-            context,
-            S_EM_SETUPCODE,
-            std::io::Cursor::new(setup_file.as_bytes()),
-        )
-        .unwrap();
+        let decrypted =
+            decrypt_setup_file(S_EM_SETUPCODE, std::io::Cursor::new(setup_file.as_bytes()))
+                .await
+                .unwrap();
 
         let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
 

src/job_thread.rs

@@ -1,203 +0,0 @@
-use std::sync::{Arc, Condvar, Mutex};
-
-use crate::context::Context;
-use crate::error::{format_err, Result};
-use crate::imap::Imap;
-
-#[derive(Debug)]
-pub struct JobThread {
-    pub name: &'static str,
-    pub folder_config_name: &'static str,
-    pub imap: Imap,
-    pub state: Arc<(Mutex<JobState>, Condvar)>,
-}
-
-#[derive(Clone, Debug, Default)]
-pub struct JobState {
-    idle: bool,
-    jobs_needed: bool,
-    suspended: bool,
-    using_handle: bool,
-}
-
-impl JobThread {
-    pub fn new(name: &'static str, folder_config_name: &'static str, imap: Imap) -> Self {
-        JobThread {
-            name,
-            folder_config_name,
-            imap,
-            state: Arc::new((Mutex::new(Default::default()), Condvar::new())),
-        }
-    }
-
-    pub fn suspend(&self, context: &Context) {
-        info!(context, "Suspending {}-thread.", self.name,);
-        {
-            self.state.0.lock().unwrap().suspended = true;
-        }
-        self.interrupt_idle(context);
-        loop {
-            let using_handle = self.state.0.lock().unwrap().using_handle;
-            if !using_handle {
-                return;
-            }
-            std::thread::sleep(std::time::Duration::from_micros(300 * 1000));
-        }
-    }
-
-    pub fn unsuspend(&self, context: &Context) {
-        info!(context, "Unsuspending {}-thread.", self.name);
-
-        let &(ref lock, ref cvar) = &*self.state.clone();
-        let mut state = lock.lock().unwrap();
-
-        state.suspended = false;
-        state.idle = true;
-        cvar.notify_one();
-    }
-
-    pub fn interrupt_idle(&self, context: &Context) {
-        {
-            self.state.0.lock().unwrap().jobs_needed = true;
-        }
-
-        info!(context, "Interrupting {}-IDLE...", self.name);
-
-        self.imap.interrupt_idle(context);
-
-        let &(ref lock, ref cvar) = &*self.state.clone();
-        let mut state = lock.lock().unwrap();
-
-        state.idle = true;
-        cvar.notify_one();
-        info!(context, "Interrupting {}-IDLE... finished", self.name);
-    }
-
-    pub async fn fetch(&mut self, context: &Context, use_network: bool) {
-        {
-            let &(ref lock, _) = &*self.state.clone();
-            let mut state = lock.lock().unwrap();
-
-            if state.suspended {
-                return;
-            }
-
-            state.using_handle = true;
-        }
-
-        if use_network {
-            if let Err(err) = self.connect_and_fetch(context).await {
-                warn!(context, "connect+fetch failed: {}, reconnect & retry", err);
-                self.imap.trigger_reconnect();
-                if let Err(err) = self.connect_and_fetch(context).await {
-                    warn!(context, "connect+fetch failed: {}", err);
-                }
-            }
-        }
-        self.state.0.lock().unwrap().using_handle = false;
-    }
-
-    async fn connect_and_fetch(&mut self, context: &Context) -> Result<()> {
-        let prefix = format!("{}-fetch", self.name);
-        self.imap.connect_configured(context)?;
-        if let Some(watch_folder) = self.get_watch_folder(context) {
-            let start = std::time::Instant::now();
-            info!(context, "{} started...", prefix);
-            let res = self
-                .imap
-                .fetch(context, &watch_folder)
-                .await
-                .map_err(Into::into);
-            let elapsed = start.elapsed().as_millis();
-            info!(context, "{} done in {:.3} ms.", prefix, elapsed);
-
-            res
-        } else {
-            Err(format_err!("WatchFolder not found: not-set"))
-        }
-    }
-
-    fn get_watch_folder(&self, context: &Context) -> Option<String> {
-        match context.sql.get_raw_config(context, self.folder_config_name) {
-            Some(name) => Some(name),
-            None => {
-                if self.folder_config_name == "configured_inbox_folder" {
-                    // initialized with old version, so has not set configured_inbox_folder
-                    Some("INBOX".to_string())
-                } else {
-                    None
-                }
-            }
-        }
-    }
-
-    pub fn idle(&self, context: &Context, use_network: bool) {
-        {
-            let &(ref lock, ref cvar) = &*self.state.clone();
-            let mut state = lock.lock().unwrap();
-
-            if state.jobs_needed {
-                info!(
-                    context,
-                    "{}-IDLE will not be started as it was interrupted while not idling.",
-                    self.name,
-                );
-                state.jobs_needed = false;
-                return;
-            }
-
-            if state.suspended {
-                while !state.idle {
-                    state = cvar.wait(state).unwrap();
-                }
-                state.idle = false;
-                return;
-            }
-
-            state.using_handle = true;
-
-            if !use_network {
-                state.using_handle = false;
-
-                while !state.idle {
-                    state = cvar.wait(state).unwrap();
-                }
-                state.idle = false;
-                return;
-            }
-        }
-
-        let prefix = format!("{}-IDLE", self.name);
-        let do_fake_idle = match self.imap.connect_configured(context) {
-            Ok(()) => {
-                if !self.imap.can_idle() {
-                    true // we have to do fake_idle
-                } else {
-                    let watch_folder = self.get_watch_folder(context);
-                    info!(context, "{} started...", prefix);
-                    let res = self.imap.idle(context, watch_folder);
-                    info!(context, "{} ended...", prefix);
-                    if let Err(err) = res {
-                        warn!(context, "{} failed: {} -> reconnecting", prefix, err);
-                        // something is borked, let's start afresh on the next occassion
-                        self.imap.disconnect(context);
-                    }
-                    false
-                }
-            }
-            Err(err) => {
-                info!(context, "{}-IDLE connection fail: {:?}", self.name, err);
-                // if the connection fails, use fake_idle to retry periodically
-                // fake_idle() will be woken up by interrupt_idle() as
-                // well so will act on maybe_network events
-                true
-            }
-        };
-        if do_fake_idle {
-            let watch_folder = self.get_watch_folder(context);
-            self.imap.fake_idle(context, watch_folder);
-        }
-
-        self.state.0.lock().unwrap().using_handle = false;
-    }
-}

src/key.rs

@@ -1,17 +1,21 @@
 //! Cryptographic key module
 
 use std::collections::BTreeMap;
+use std::fmt;
 use std::io::Cursor;
-use std::path::Path;
 
+use async_trait::async_trait;
+use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
+use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::*;
-use crate::sql::Sql;
+use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
+use crate::sql;
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
@@ -19,11 +23,24 @@ pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 
 /// Error type for deltachat key handling.
 #[derive(Debug, thiserror::Error)]
+#[non_exhaustive]
 pub enum Error {
     #[error("Could not decode base64")]
     Base64Decode(#[from] base64::DecodeError),
-    #[error("rPGP error: {0}")]
-    PgpError(#[from] pgp::errors::Error),
+    #[error("rPGP error: {}", _0)]
+    Pgp(#[from] pgp::errors::Error),
+    #[error("Failed to generate PGP key: {}", _0)]
+    Keygen(#[from] crate::pgp::PgpKeygenError),
+    #[error("Failed to load key: {}", _0)]
+    LoadKey(#[from] sql::Error),
+    #[error("Failed to save generated key: {}", _0)]
+    StoreKey(#[from] SaveKeyError),
+    #[error("No address configured")]
+    NoConfiguredAddr,
+    #[error("Configured address is invalid: {}", _0)]
+    InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[error("no data provided")]
+    Empty,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -33,8 +50,9 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// This trait is implemented for rPGP's [SignedPublicKey] and
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
-pub trait DcKey: Serialize + Deserializable {
-    type KeyType: Serialize + Deserializable;
+#[async_trait]
+pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
+    type KeyType: Serialize + Deserializable + KeyTrait + Clone;
 
     /// Create a key from some bytes.
     fn from_slice(bytes: &[u8]) -> Result<Self::KeyType> {
@@ -51,244 +69,198 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
-    /// Serialise the key to a base64 string.
-    fn to_base64(&self) -> String {
+    /// Create a key from an ASCII-armored string.
+    ///
+    /// Returns the key and a map of any headers which might have been set in
+    /// the ASCII-armored representation.
+    fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
+        let bytes = data.as_bytes();
+        Self::KeyType::from_armor_single(Cursor::new(bytes)).map_err(Error::Pgp)
+    }
+
+    /// Load the users' default key from the database.
+    async fn load_self(context: &Context) -> Result<Self::KeyType>;
+
+    /// Serialise the key as bytes.
+    fn to_bytes(&self) -> Vec<u8> {
         // Not using Serialize::to_bytes() to make clear *why* it is
         // safe to ignore this error.
         // Because we write to a Vec<u8> the io::Write impls never
         // fail and we can hide this error.
         let mut buf = Vec::new();
         self.to_writer(&mut buf).unwrap();
-        base64::encode(&buf)
+        buf
+    }
+
+    /// Serialise the key to a base64 string.
+    fn to_base64(&self) -> String {
+        base64::encode(&DcKey::to_bytes(self))
+    }
+
+    /// Serialise the key to ASCII-armored representation.
+    ///
+    /// Each header line must be terminated by `\r\n`.  Only allows setting one
+    /// header as a simplification since that's the only way it's used so far.
+    // Since .to_armored_string() are actual methods on SignedPublicKey and
+    // SignedSecretKey we can not generically implement this.
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String;
+
+    /// The fingerprint for the key.
+    fn fingerprint(&self) -> Fingerprint {
+        Fingerprint::new(KeyTrait::fingerprint(self)).expect("Invalid fingerprint from rpgp")
     }
 }
 
+#[async_trait]
 impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
-}
-
-impl DcKey for SignedSecretKey {
-    type KeyType = SignedSecretKey;
-}
-
-/// Cryptographic key
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum Key {
-    Public(SignedPublicKey),
-    Secret(SignedSecretKey),
-}
-
-impl From<SignedPublicKey> for Key {
-    fn from(key: SignedPublicKey) -> Self {
-        Key::Public(key)
-    }
-}
-
-impl From<SignedSecretKey> for Key {
-    fn from(key: SignedSecretKey) -> Self {
-        Key::Secret(key)
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl Key {
-    pub fn is_public(&self) -> bool {
-        match self {
-            Key::Public(_) => true,
-            Key::Secret(_) => false,
-        }
-    }
-
-    pub fn is_secret(&self) -> bool {
-        !self.is_public()
-    }
-
-    pub fn from_slice(bytes: &[u8], key_type: KeyType) -> Option<Self> {
-        if bytes.is_empty() {
-            return None;
-        }
-        let res: std::result::Result<Key, _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-            KeyType::Private => SignedSecretKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-        };
 
+    async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        let res: std::result::Result<Vec<u8>, _> = context
+            .sql
+            .query_value(
+                r#"
+            SELECT public_key
+              FROM keypairs
+             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
+               AND is_default=1;
+            "#,
+                paramsx![],
+            )
+            .await;
         match res {
-            Ok(key) => Some(key),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
+            Ok(bytes) => Self::from_slice(&bytes),
+            Err(sql::Error::Sqlx(sqlx::Error::RowNotFound)) => {
+                let keypair = generate_keypair(context).await?;
+                Ok(keypair.public)
             }
+            Err(err) => Err(err.into()),
         }
     }
 
-    pub fn from_armored_string(
-        data: &str,
-        key_type: KeyType,
-    ) -> Option<(Self, BTreeMap<String, String>)> {
-        let bytes = data.as_bytes();
-        let res: std::result::Result<(Key, _), _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-            KeyType::Private => SignedSecretKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-        };
-
-        match res {
-            Ok(res) => Some(res),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn from_self_public(
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> Option<Self> {
-        let addr = self_addr.as_ref();
-
-        sql.query_get_value(
-            context,
-            "SELECT public_key FROM keypairs WHERE addr=? AND is_default=1;",
-            &[addr],
-        )
-        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Public))
-    }
-
-    pub fn from_self_private(
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> Option<Self> {
-        sql.query_get_value(
-            context,
-            "SELECT private_key FROM keypairs WHERE addr=? AND is_default=1;",
-            &[self_addr.as_ref()],
-        )
-        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Private))
-    }
-
-    pub fn to_bytes(&self) -> Vec<u8> {
-        match self {
-            Key::Public(k) => k.to_bytes().unwrap_or_default(),
-            Key::Secret(k) => k.to_bytes().unwrap_or_default(),
-        }
-    }
-
-    pub fn verify(&self) -> bool {
-        match self {
-            Key::Public(k) => k.verify().is_ok(),
-            Key::Secret(k) => k.verify().is_ok(),
-        }
-    }
-
-    pub fn to_base64(&self) -> String {
-        let buf = self.to_bytes();
-        base64::encode(&buf)
-    }
-
-    pub fn to_armored_string(
-        &self,
-        headers: Option<&BTreeMap<String, String>>,
-    ) -> pgp::errors::Result<String> {
-        match self {
-            Key::Public(k) => k.to_armored_string(headers),
-            Key::Secret(k) => k.to_armored_string(headers),
-        }
-    }
-
-    /// Each header line must be terminated by `\r\n`
-    pub fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to ignore this error.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.
         let headers = header.map(|(key, value)| {
             let mut m = BTreeMap::new();
             m.insert(key.to_string(), value.to_string());
             m
         });
-
-        self.to_armored_string(headers.as_ref())
-            .expect("failed to serialize key")
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
     }
+}
 
-    pub fn write_asc_to_file(
-        &self,
-        file: impl AsRef<Path>,
-        context: &Context,
-    ) -> std::io::Result<()> {
-        let file_content = self.to_asc(None).into_bytes();
+#[async_trait]
+impl DcKey for SignedSecretKey {
+    type KeyType = SignedSecretKey;
 
-        let res = dc_write_file(context, &file, &file_content);
-        if res.is_err() {
-            error!(context, "Cannot write key to {}", file.as_ref().display());
-        }
-        res
-    }
-
-    pub fn fingerprint(&self) -> String {
-        match self {
-            Key::Public(k) => hex::encode_upper(k.fingerprint()),
-            Key::Secret(k) => hex::encode_upper(k.fingerprint()),
-        }
-    }
-
-    pub fn formatted_fingerprint(&self) -> String {
-        let rawhex = self.fingerprint();
-        dc_format_fingerprint(&rawhex)
-    }
-
-    pub fn split_key(&self) -> Option<Key> {
-        match self {
-            Key::Public(_) => None,
-            Key::Secret(k) => {
-                let pub_key = k.public_key();
-                pub_key.sign(k, || "".into()).map(Key::Public).ok()
+    async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        let res: std::result::Result<Vec<u8>, _> = context
+            .sql
+            .query_value(
+                r#"
+            SELECT private_key
+              FROM keypairs
+             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
+               AND is_default=1;
+            "#,
+                paramsx![],
+            )
+            .await;
+        match res {
+            Ok(bytes) => Self::from_slice(&bytes),
+            Err(sql::Error::Sqlx(sqlx::Error::RowNotFound)) => {
+                let keypair = generate_keypair(context).await?;
+                Ok(keypair.secret)
             }
+            Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to do these unwraps.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.  The string is always ASCII.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
+}
+
+/// Deltachat extension trait for secret keys.
+///
+/// Provides some convenience wrappers only applicable to [SignedSecretKey].
+pub trait DcSecretKey {
+    /// Create a public key from a private one.
+    fn split_public_key(&self) -> Result<SignedPublicKey>;
+}
+
+impl DcSecretKey for SignedSecretKey {
+    fn split_public_key(&self) -> Result<SignedPublicKey> {
+        self.verify()?;
+        let unsigned_pubkey = SecretKeyTrait::public_key(self);
+        let signed_pubkey = unsigned_pubkey.sign(self, || "".into())?;
+        Ok(signed_pubkey)
+    }
+}
+
+async fn generate_keypair(context: &Context) -> Result<KeyPair> {
+    let addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await
+        .ok_or_else(|| Error::NoConfiguredAddr)?;
+    let addr = EmailAddress::new(&addr)?;
+    let _guard = context.generating_key_mutex.lock().await;
+
+    // Check if the key appeared while we were waiting on the lock.
+    let res: std::result::Result<(Vec<u8>, Vec<u8>), _> = context
+        .sql
+        .query_row(
+            r#"
+        SELECT public_key, private_key
+          FROM keypairs
+         WHERE addr=?
+           AND is_default=1;
+        "#,
+            paramsx![addr.to_string()],
+        )
+        .await;
+    match res {
+        Ok((pub_bytes, sec_bytes)) => Ok(KeyPair {
+            addr,
+            public: SignedPublicKey::from_slice(&pub_bytes)?,
+            secret: SignedSecretKey::from_slice(&sec_bytes)?,
+        }),
+        Err(sql::Error::Sqlx(sqlx::Error::RowNotFound)) => {
+            let start = std::time::Instant::now();
+            let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
+                .unwrap_or_default();
+            info!(context, "Generating keypair with type {}", keytype);
+            let keypair =
+                async_std::task::spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
+                    .await?;
+            store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
+            info!(
+                context,
+                "Keypair generated in {:.3}s.",
+                start.elapsed().as_secs()
+            );
+            Ok(keypair)
+        }
+        Err(err) => Err(err.into()),
+    }
 }
 
 /// Use of a [KeyPair] for encryption or decryption.
@@ -333,103 +305,130 @@ impl SaveKeyError {
 /// same key again overwrites it.
 ///
 /// [Config::ConfiguredAddr]: crate::config::Config::ConfiguredAddr
-pub fn store_self_keypair(
+pub async fn store_self_keypair(
     context: &Context,
     keypair: &KeyPair,
     default: KeyPairUse,
 ) -> std::result::Result<(), SaveKeyError> {
     // Everything should really be one transaction, more refactoring
     // is needed for that.
-    let public_key = keypair
-        .public
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise public key", err))?;
-    let secret_key = keypair
-        .secret
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise secret key", err))?;
+    let public_key = DcKey::to_bytes(&keypair.public);
+    let secret_key = DcKey::to_bytes(&keypair.secret);
     context
         .sql
         .execute(
             "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
-            params![public_key, secret_key],
+            paramsx![&public_key, &secret_key],
         )
+        .await
         .map_err(|err| SaveKeyError::new("failed to remove old use of key", err))?;
     if default == KeyPairUse::Default {
         context
             .sql
-            .execute("UPDATE keypairs SET is_default=0;", params![])
+            .execute("UPDATE keypairs SET is_default=0;", paramsx![])
+            .await
             .map_err(|err| SaveKeyError::new("failed to clear default", err))?;
     }
     let is_default = match default {
-        KeyPairUse::Default => true,
-        KeyPairUse::ReadOnly => false,
+        KeyPairUse::Default => true as i32,
+        KeyPairUse::ReadOnly => false as i32,
     };
+
+    let addr = keypair.addr.to_string();
+    let t = time();
+
+    let params = paramsx![addr, is_default, public_key, secret_key, t];
     context
         .sql
         .execute(
             "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
                 VALUES (?,?,?,?,?);",
-            params![
-                keypair.addr.to_string(),
-                is_default as i32,
-                public_key,
-                secret_key,
-                time()
-            ],
+            params,
         )
-        .map(|_| ())
-        .map_err(|err| SaveKeyError::new("failed to insert keypair", err))
+        .await
+        .map_err(|err| SaveKeyError::new("failed to insert keypair", err))?;
+
+    Ok(())
 }
 
-/// Make a fingerprint human-readable, in hex format.
-pub fn dc_format_fingerprint(fingerprint: &str) -> String {
-    // split key into chunks of 4 with space, and 20 newline
-    let mut res = String::new();
+/// A key fingerprint
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct Fingerprint(Vec<u8>);
 
-    for (i, c) in fingerprint.chars().enumerate() {
-        if i > 0 && i % 20 == 0 {
-            res += "\n";
-        } else if i > 0 && i % 4 == 0 {
-            res += " ";
+impl Fingerprint {
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+        match v.len() {
+            20 => Ok(Fingerprint(v)),
+            _ => Err(FingerprintError::WrongLength),
         }
-
-        res += &c.to_string();
     }
 
-    res
+    /// Make a hex string from the fingerprint.
+    ///
+    /// Use [std::fmt::Display] or [ToString::to_string] to get a
+    /// human-readable formatted string.
+    pub fn hex(&self) -> String {
+        hex::encode_upper(&self.0)
+    }
 }
 
-/// Bring a human-readable or otherwise formatted fingerprint back to the 40-characters-uppercase-hex format.
-pub fn dc_normalize_fingerprint(fp: &str) -> String {
-    fp.to_uppercase()
-        .chars()
-        .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
-        .collect()
+/// Make a human-readable fingerprint.
+impl fmt::Display for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Split key into chunks of 4 with space and newline at 20 chars
+        for (i, c) in self.hex().chars().enumerate() {
+            if i > 0 && i % 20 == 0 {
+                writeln!(f)?;
+            } else if i > 0 && i % 4 == 0 {
+                write!(f, " ")?;
+            }
+            write!(f, "{}", c)?;
+        }
+        Ok(())
+    }
+}
+
+/// Parse a human-readable or otherwise formatted fingerprint.
+impl std::str::FromStr for Fingerprint {
+    type Err = FingerprintError;
+
+    fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
+        let hex_repr: String = input
+            .to_uppercase()
+            .chars()
+            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .collect();
+        let v: Vec<u8> = hex::decode(hex_repr)?;
+        let fp = Fingerprint::new(v)?;
+        Ok(fp)
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use std::convert::TryFrom;
 
+    use std::error::Error;
+
+    use async_std::sync::Arc;
     use lazy_static::lazy_static;
 
     lazy_static! {
         static ref KEYPAIR: KeyPair = alice_keypair();
     }
 
-    #[test]
-    fn test_normalize_fingerprint() {
-        let fingerprint = dc_normalize_fingerprint(" 1234  567890 \n AbcD abcdef ABCDEF ");
-
-        assert_eq!(fingerprint, "1234567890ABCDABCDEFABCDEF");
-    }
-
     #[test]
     fn test_from_armored_string() {
-        let (private_key, _) = Key::from_armored_string(
+        let (private_key, _) = SignedSecretKey::from_asc(
             "-----BEGIN PGP PRIVATE KEY BLOCK-----
 
 xcLYBF0fgz4BCADnRUV52V4xhSsU56ZaAn3+3oG86MZhXy4X8w14WZZDf0VJGeTh
@@ -487,99 +486,149 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 7yPJeQ==
 =KZk/
 -----END PGP PRIVATE KEY BLOCK-----",
-            KeyType::Private,
         )
-        .expect("failed to decode"); // NOTE: if you take out the ===GU1/ part, everything passes!
-        let binary = private_key.to_bytes();
-        Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        .expect("failed to decode");
+        let binary = DcKey::to_bytes(&private_key);
+        SignedSecretKey::from_slice(&binary).expect("invalid private key");
     }
 
     #[test]
-    fn test_format_fingerprint() {
-        let fingerprint = dc_format_fingerprint("1234567890ABCDABCDEFABCDEF1234567890ABCD");
+    fn test_asc_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedPublicKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
 
-        assert_eq!(
-            fingerprint,
-            "1234 5678 90AB CDAB CDEF\nABCD EF12 3456 7890 ABCD"
-        );
+        let key = KEYPAIR.secret.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedSecretKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
     }
 
     #[test]
     fn test_from_slice_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+        let public_key = KEYPAIR.public.clone();
+        let private_key = KEYPAIR.secret.clone();
 
-        let binary = public_key.to_bytes();
-        let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
+        let binary = DcKey::to_bytes(&public_key);
+        let public_key2 = SignedPublicKey::from_slice(&binary).expect("invalid public key");
         assert_eq!(public_key, public_key2);
 
-        let binary = private_key.to_bytes();
-        let private_key2 = Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        let binary = DcKey::to_bytes(&private_key);
+        let private_key2 = SignedSecretKey::from_slice(&binary).expect("invalid private key");
         assert_eq!(private_key, private_key2);
     }
 
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-
         for i in 0..4096 {
             bad_data[i] = (i & 0xff) as u8;
         }
-
         for j in 0..(4096 / 40) {
-            let bad_key = Key::from_slice(
-                &bad_data[j..j + 4096 / 2 + j],
-                if 0 != j & 1 {
-                    KeyType::Public
-                } else {
-                    KeyType::Private
-                },
-            );
-            assert!(bad_key.is_none());
+            let slice = &bad_data[j..j + 4096 / 2 + j];
+            assert!(SignedPublicKey::from_slice(slice).is_err());
+            assert!(SignedSecretKey::from_slice(slice).is_err());
         }
     }
 
     #[test]
-    fn test_ascii_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+    fn test_base64_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let base64 = key.to_base64();
+        let key2 = SignedPublicKey::from_base64(&base64).unwrap();
+        assert_eq!(key, key2);
+    }
 
-        let s = public_key.to_armored_string(None).unwrap();
-        let (public_key2, _) =
-            Key::from_armored_string(&s, KeyType::Public).expect("invalid public key");
-        assert_eq!(public_key, public_key2);
+    #[async_std::test]
+    async fn test_load_self_existing() {
+        let alice = alice_keypair();
+        let t = TestContext::new().await;
+        t.configure_alice().await;
+        let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
+        assert_eq!(alice.public, pubkey);
+        let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
+        assert_eq!(alice.secret, seckey);
+    }
 
-        let s = private_key.to_armored_string(None).unwrap();
-        println!("{}", &s);
-        let (private_key2, _) =
-            Key::from_armored_string(&s, KeyType::Private).expect("invalid private key");
-        assert_eq!(private_key, private_key2);
+    #[async_std::test]
+    async fn test_load_self_generate_public() {
+        let t = TestContext::new().await;
+        t.ctx
+            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
+            .unwrap();
+        let key = SignedPublicKey::load_self(&t.ctx).await;
+        assert!(key.is_ok());
+    }
+
+    #[async_std::test]
+    async fn test_load_self_generate_secret() {
+        let t = TestContext::new().await;
+        t.ctx
+            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
+            .unwrap();
+        let key = SignedSecretKey::load_self(&t.ctx).await;
+        assert!(key.is_ok());
+    }
+
+    #[async_std::test]
+    async fn test_load_self_generate_concurrent() {
+        use std::thread;
+
+        let t = TestContext::new().await;
+        t.ctx
+            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
+            .unwrap();
+        let ctx = t.ctx.clone();
+        let ctx0 = ctx.clone();
+        let thr0 =
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
+        let ctx1 = ctx.clone();
+        let thr1 =
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
+        let res0 = thr0.join().unwrap();
+        let res1 = thr1.join().unwrap();
+        assert_eq!(res0.unwrap(), res1.unwrap());
     }
 
     #[test]
     fn test_split_key() {
-        let private_key = Key::from(KEYPAIR.secret.clone());
-        let public_wrapped = private_key.split_key().unwrap();
-        let public = SignedPublicKey::try_from(public_wrapped).unwrap();
-        assert_eq!(public.primary_key, KEYPAIR.public.primary_key);
+        let pubkey = KEYPAIR.secret.split_public_key().unwrap();
+        assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
-    #[test]
-    fn test_save_self_key_twice() {
+    #[async_std::test]
+    async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.
-        let t = dummy_context();
-        let nrows = || {
-            t.ctx
+        let t = TestContext::new().await;
+        let ctx = Arc::new(t.ctx);
+
+        let ctx1 = ctx.clone();
+        let nrows = || async {
+            let val: i32 = ctx1
                 .sql
-                .query_get_value::<_, u32>(&t.ctx, "SELECT COUNT(*) FROM keypairs;", params![])
-                .unwrap()
+                .query_value("SELECT COUNT(*) FROM keypairs;", paramsx![])
+                .await
+                .unwrap();
+            val as usize
         };
-        assert_eq!(nrows(), 0);
-        store_self_keypair(&t.ctx, &KEYPAIR, KeyPairUse::Default).unwrap();
-        assert_eq!(nrows(), 1);
-        store_self_keypair(&t.ctx, &KEYPAIR, KeyPairUse::Default).unwrap();
-        assert_eq!(nrows(), 1);
+        assert_eq!(nrows().await, 0);
+        store_self_keypair(&ctx, &KEYPAIR, KeyPairUse::Default)
+            .await
+            .unwrap();
+        assert_eq!(nrows().await, 1);
+        store_self_keypair(&ctx, &KEYPAIR, KeyPairUse::Default)
+            .await
+            .unwrap();
+        assert_eq!(nrows().await, 1);
     }
 
     // Convenient way to create a new key if you need one, run with
@@ -602,4 +651,49 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     //     )
     //     .unwrap();
     // }
+
+    #[test]
+    fn test_fingerprint_from_str() {
+        let res = Fingerprint::new(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+
+        let fp: Fingerprint = "0102030405060708090A0B0c0d0e0F1011121314".parse().unwrap();
+        assert_eq!(fp, res);
+
+        let fp: Fingerprint = "zzzz 0102 0304 0506\n0708090a0b0c0D0E0F1011121314 yyy"
+            .parse()
+            .unwrap();
+        assert_eq!(fp, res);
+
+        let err = "1".parse::<Fingerprint>().err().unwrap();
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
+    }
+
+    #[test]
+    fn test_fingerprint_hex() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(fp.hex(), "0102040810204080FF0A0B0C0D0E0F1011121314");
+    }
+
+    #[test]
+    fn test_fingerprint_to_string() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(
+            fp.to_string(),
+            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF 1314"
+        );
+    }
 }

src/keyring.rs

@@ -1,45 +1,92 @@
-use std::borrow::Cow;
+//! Keyring to perform rpgp operations with.
+
+use anyhow::Result;
 
-use crate::constants::KeyType;
 use crate::context::Context;
-use crate::key::Key;
-use crate::sql::Sql;
+use crate::key::{self, DcKey};
 
-#[derive(Default, Clone, Debug)]
-pub struct Keyring<'a> {
-    keys: Vec<Cow<'a, Key>>,
+/// An in-memory keyring.
+///
+/// Instances are usually constructed just for the rpgp operation and
+/// short-lived.
+#[derive(Clone, Debug, Default)]
+pub struct Keyring<T>
+where
+    T: DcKey,
+{
+    keys: Vec<T>,
 }
 
-impl<'a> Keyring<'a> {
-    pub fn add_owned(&mut self, key: Key) {
-        self.add(Cow::Owned(key))
+impl<T> Keyring<T>
+where
+    T: DcKey<KeyType = T>,
+{
+    /// New empty keyring.
+    pub fn new() -> Keyring<T> {
+        Keyring { keys: Vec::new() }
     }
 
-    pub fn add_ref(&mut self, key: &'a Key) {
-        self.add(Cow::Borrowed(key))
+    /// Create a new keyring with the the user's secret key loaded.
+    pub async fn new_self(context: &Context) -> Result<Keyring<T>, key::Error> {
+        let mut keyring: Keyring<T> = Keyring::new();
+        keyring.load_self(context).await?;
+        Ok(keyring)
     }
 
-    fn add(&mut self, key: Cow<'a, Key>) {
+    /// Load the user's key into the keyring.
+    pub async fn load_self(&mut self, context: &Context) -> Result<(), key::Error> {
+        self.add(T::load_self(context).await?);
+        Ok(())
+    }
+
+    /// Add a key to the keyring.
+    pub fn add(&mut self, key: T) {
         self.keys.push(key);
     }
 
-    pub fn keys(&self) -> &[Cow<'a, Key>] {
-        &self.keys
+    pub fn len(&self) -> usize {
+        self.keys.len()
     }
 
-    pub fn load_self_private_for_decrypting(
-        &mut self,
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> bool {
-        sql.query_get_value(
-            context,
-            "SELECT private_key FROM keypairs ORDER BY addr=? DESC, is_default DESC;",
-            &[self_addr.as_ref()],
-        )
-        .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Private))
-        .map(|key| self.add_owned(key))
-        .is_some()
+    pub fn is_empty(&self) -> bool {
+        self.keys.is_empty()
+    }
+
+    /// A vector with reference to all the keys in the keyring.
+    pub fn keys(&self) -> &[T] {
+        &self.keys
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::key::{SignedPublicKey, SignedSecretKey};
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_keyring_add_keys() {
+        let alice = alice_keypair();
+        let mut pub_ring: Keyring<SignedPublicKey> = Keyring::new();
+        pub_ring.add(alice.public.clone());
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let mut sec_ring: Keyring<SignedSecretKey> = Keyring::new();
+        sec_ring.add(alice.secret.clone());
+        assert_eq!(sec_ring.keys(), [alice.secret]);
+    }
+
+    #[async_std::test]
+    async fn test_keyring_load_self() {
+        // new_self() implies load_self()
+        let t = TestContext::new().await;
+        t.configure_alice().await;
+        let alice = alice_keypair();
+
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -6,19 +6,22 @@
 extern crate num_derive;
 #[macro_use]
 extern crate smallvec;
-#[macro_use]
-extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 #[macro_use]
 pub mod log;
 #[macro_use]
 pub mod error;
 
+#[cfg(feature = "internals")]
+#[macro_use]
+pub mod sql;
+#[cfg(not(feature = "internals"))]
+#[macro_use]
+mod sql;
+
 pub mod headerdef;
 
 pub(crate) mod events;
@@ -36,9 +39,9 @@ pub mod context;
 mod e2ee;
 mod imap;
 pub mod imex;
+mod scheduler;
 #[macro_use]
 pub mod job;
-mod job_thread;
 pub mod key;
 mod keyring;
 pub mod location;
@@ -56,7 +59,6 @@ pub mod qr;
 pub mod securejoin;
 mod simplify;
 mod smtp;
-pub mod sql;
 pub mod stock;
 mod token;
 #[macro_use]

src/location.rs

@@ -1,5 +1,6 @@
 //! Location handling
 
+use async_std::prelude::*;
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
@@ -10,11 +11,10 @@ use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
 use crate::events::Event;
-use crate::job::{self, job_action_exists, job_add, Job};
+use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
-use crate::sql;
 use crate::stock::StockMessage;
 
 /// Location record
@@ -38,6 +38,34 @@ impl Location {
     }
 }
 
+impl<'a> sqlx::FromRow<'a, sqlx::sqlite::SqliteRow> for Location {
+    fn from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Self, sqlx::Error> {
+        use sqlx::Row;
+
+        let msg_id = row.try_get::<i32, _>("msg_id")? as u32;
+        let txt: String = row.try_get("txt")?;
+        let marker = if msg_id != 0 && is_marker(&txt) {
+            Some(txt)
+        } else {
+            None
+        };
+        let loc = Location {
+            location_id: row.try_get::<i32, _>("id")? as u32,
+            latitude: row.try_get("latitude")?,
+            longitude: row.try_get("longitude")?,
+            accuracy: row.try_get("accuracy")?,
+            timestamp: row.try_get("timestamp")?,
+            independent: row.try_get::<i32, _>("independent")? as u32,
+            msg_id,
+            contact_id: row.try_get::<i32, _>("from_id")? as u32,
+            chat_id: row.try_get("chat_id")?,
+            marker,
+        };
+
+        Ok(loc)
+    }
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct Kml {
     pub addr: Option<String>,
@@ -191,114 +219,135 @@ impl Kml {
 }
 
 // location streaming
-pub fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
+pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
     if !(seconds < 0 || chat_id.is_special()) {
-        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
-        if sql::execute(
-            context,
-            &context.sql,
-            "UPDATE chats    \
-             SET locations_send_begin=?,        \
-             locations_send_until=?  \
-             WHERE id=?",
-            params![
-                if 0 != seconds { now } else { 0 },
-                if 0 != seconds { now + seconds } else { 0 },
-                chat_id,
-            ],
-        )
-        .is_ok()
+        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id).await;
+        if context
+            .sql
+            .execute(
+                r#"
+UPDATE chats
+  SET locations_send_begin=?,
+      locations_send_until=?
+  WHERE id=?
+"#,
+                paramsx![
+                    if 0 != seconds { now } else { 0 },
+                    if 0 != seconds { now + seconds } else { 0 },
+                    chat_id
+                ],
+            )
+            .await
+            .is_ok()
         {
             if 0 != seconds && !is_sending_locations_before {
                 let mut msg = Message::new(Viewtype::Text);
-                msg.text =
-                    Some(context.stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0));
-                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
-                chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
-            } else if 0 == seconds && is_sending_locations_before {
-                let stock_str =
-                    context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                chat::add_info_msg(context, chat_id, stock_str);
-            }
-            context.call_cb(Event::ChatModified(chat_id));
-            if 0 != seconds {
-                schedule_MAYBE_SEND_LOCATIONS(context, false);
-                job_add(
-                    context,
-                    job::Action::MaybeSendLocationsEnded,
-                    chat_id.to_u32() as i32,
-                    Params::new(),
-                    seconds + 1,
+                msg.text = Some(
+                    context
+                        .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
+                        .await,
                 );
+                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
+                chat::send_msg(context, chat_id, &mut msg)
+                    .await
+                    .unwrap_or_default();
+            } else if 0 == seconds && is_sending_locations_before {
+                let stock_str = context
+                    .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
+                    .await;
+                chat::add_info_msg(context, chat_id, stock_str).await;
+            }
+            context.emit_event(Event::ChatModified(chat_id));
+            if 0 != seconds {
+                schedule_maybe_send_locations(context, false).await;
+                job::add(
+                    context,
+                    job::Job::new(
+                        job::Action::MaybeSendLocationsEnded,
+                        chat_id.to_u32(),
+                        Params::new(),
+                        seconds + 1,
+                    ),
+                )
+                .await;
             }
         }
     }
 }
 
-#[allow(non_snake_case)]
-fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, force_schedule: bool) {
-    if force_schedule || !job_action_exists(context, job::Action::MaybeSendLocations) {
-        job_add(
+async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) {
+    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await {
+        job::add(
             context,
-            job::Action::MaybeSendLocations,
-            0,
-            Params::new(),
-            60,
-        );
+            job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
+        )
+        .await;
     };
 }
 
-pub fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
+pub async fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
     context
         .sql
         .exists(
             "SELECT id  FROM chats  WHERE (? OR id=?)   AND locations_send_until>?;",
-            params![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
+            paramsx![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
         )
+        .await
         .unwrap_or_default()
 }
 
-pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
+pub async fn set(
+    context: &Context,
+    latitude: f64,
+    longitude: f64,
+    accuracy: f64,
+) -> Result<bool, Error> {
     if latitude == 0.0 && longitude == 0.0 {
-        return true;
+        return Ok(true);
     }
+
     let mut continue_streaming = false;
+    let pool = context.sql.get_pool().await?;
+    let mut rows = sqlx::query_as("SELECT id FROM chats WHERE locations_send_until>?;")
+        .bind(time())
+        .fetch(&pool);
 
-    if let Ok(chats) = context.sql.query_map(
-        "SELECT id FROM chats WHERE locations_send_until>?;",
-        params![time()],
-        |row| row.get::<_, i32>(0),
-        |chats| chats.collect::<Result<Vec<_>, _>>().map_err(Into::into),
-    ) {
-        for chat_id in chats {
-            if let Err(err) = context.sql.execute(
-                    "INSERT INTO locations  \
-                     (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);",
-                    params![
-                        latitude,
-                        longitude,
-                        accuracy,
-                        time(),
-                        chat_id,
-                        DC_CONTACT_ID_SELF,
-                    ]
-            ) {
-                warn!(context, "failed to store location {:?}", err);
-            } else {
-                continue_streaming = true;
-            }
+    while let Some(row) = rows.next().await {
+        let (chat_id,): (i64,) = row?;
+
+        if let Err(err) = context
+            .sql
+            .execute(
+                "INSERT INTO locations  \
+             (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);",
+                paramsx![
+                    latitude,
+                    longitude,
+                    accuracy,
+                    time(),
+                    chat_id,
+                    DC_CONTACT_ID_SELF as i32
+                ],
+            )
+            .await
+        {
+            warn!(context, "failed to store location {:?}", err);
+        } else {
+            continue_streaming = true;
         }
-        if continue_streaming {
-            context.call_cb(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
-        };
-        schedule_MAYBE_SEND_LOCATIONS(context, false);
     }
 
-    continue_streaming
+    if continue_streaming {
+        context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+    };
+
+    schedule_maybe_send_locations(context, false).await;
+
+    Ok(continue_streaming)
 }
 
-pub fn get_range(
+pub async fn get_range(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
@@ -308,16 +357,21 @@ pub fn get_range(
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
+
     context
         .sql
-        .query_map(
-            "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
-             COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt \
-             FROM locations l  LEFT JOIN msgs m ON l.id=m.location_id  WHERE (? OR l.chat_id=?) \
-             AND (? OR l.from_id=?) \
-             AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
-             ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
-            params![
+        .query_rows(
+            r#"
+SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent,
+    COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt
+  FROM locations l
+  LEFT JOIN msgs m ON l.id=m.location_id
+    WHERE (? OR l.chat_id=?)
+      AND (? OR l.from_id=?)
+      AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?))
+  ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;
+"#,
+            paramsx![
                 if chat_id.is_unset() { 1 } else { 0 },
                 chat_id,
                 if contact_id == 0 { 1 } else { 0 },
@@ -325,37 +379,8 @@ pub fn get_range(
                 timestamp_from,
                 timestamp_to,
             ],
-            |row| {
-                let msg_id = row.get(6)?;
-                let txt: String = row.get(9)?;
-                let marker = if msg_id != 0 && is_marker(&txt) {
-                    Some(txt)
-                } else {
-                    None
-                };
-                let loc = Location {
-                    location_id: row.get(0)?,
-                    latitude: row.get(1)?,
-                    longitude: row.get(2)?,
-                    accuracy: row.get(3)?,
-                    timestamp: row.get(4)?,
-                    independent: row.get(5)?,
-                    msg_id,
-                    contact_id: row.get(7)?,
-                    chat_id: row.get(8)?,
-                    marker,
-                };
-                Ok(loc)
-            },
-            |locations| {
-                let mut ret = Vec::new();
-
-                for location in locations {
-                    ret.push(location?);
-                }
-                Ok(ret)
-            },
         )
+        .await
         .unwrap_or_default()
 }
 
@@ -364,28 +389,27 @@ fn is_marker(txt: &str) -> bool {
 }
 
 /// Deletes all locations from the database.
-pub fn delete_all(context: &Context) -> Result<(), Error> {
-    sql::execute(context, &context.sql, "DELETE FROM locations;", params![])?;
-    context.call_cb(Event::LocationChanged(None));
+pub async fn delete_all(context: &Context) -> Result<(), Error> {
+    context
+        .sql
+        .execute("DELETE FROM locations;", paramsx![])
+        .await?;
+    context.emit_event(Event::LocationChanged(None));
     Ok(())
 }
 
-pub fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
+pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
     let mut last_added_location_id = 0;
 
     let self_addr = context
         .get_config(Config::ConfiguredAddr)
+        .await
         .unwrap_or_default();
 
-    let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
+    let (locations_send_begin, locations_send_until, locations_last_sent): (i64, i64, i64) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
-        params![chat_id], |row| {
-            let send_begin: i64 = row.get(0)?;
-            let send_until: i64 = row.get(1)?;
-            let last_sent: i64 = row.get(2)?;
-
-            Ok((send_begin, send_until, last_sent))
-        })?;
+        paramsx![chat_id]
+    ).await?;
 
     let now = time();
     let mut location_count = 0;
@@ -396,40 +420,41 @@ pub fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Erro
             self_addr,
         );
 
-        context.sql.query_map(
-            "SELECT id, latitude, longitude, accuracy, timestamp \
-             FROM locations  WHERE from_id=? \
-             AND timestamp>=? \
-             AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
-             AND independent=0 \
-             GROUP BY timestamp \
-             ORDER BY timestamp;",
-            params![DC_CONTACT_ID_SELF, locations_send_begin, locations_last_sent, DC_CONTACT_ID_SELF],
-            |row| {
-                let location_id: i32 = row.get(0)?;
-                let latitude: f64 = row.get(1)?;
-                let longitude: f64 = row.get(2)?;
-                let accuracy: f64 = row.get(3)?;
-                let timestamp = get_kml_timestamp(row.get(4)?);
+        let pool = context.sql.get_pool().await?;
+        let mut rows = sqlx::query_as(
+            r#"
+SELECT id, latitude, longitude, accuracy, timestamp
+  FROM locations
+  WHERE from_id=?
+    AND timestamp>=?
+    AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?))
+    AND independent=0
+  GROUP BY timestamp
+  ORDER BY timestamp;
+"#,
+        )
+        .bind(DC_CONTACT_ID_SELF as i32)
+        .bind(locations_send_begin)
+        .bind(locations_last_sent)
+        .bind(DC_CONTACT_ID_SELF as i32)
+        .fetch(&pool);
+
+        while let Some(row) = rows.next().await {
+            let (location_id, latitude, longitude, accuracy, timestamp): (i32, f64, f64, f64, i64) =
+                row?;
+            let timestamp = get_kml_timestamp(timestamp);
+
+            ret += &format!(
+                "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
+                timestamp,
+                accuracy,
+                longitude,
+                latitude
+            );
+            location_count += 1;
+            last_added_location_id = location_id as u32;
+        }
 
-                Ok((location_id, latitude, longitude, accuracy, timestamp))
-            },
-            |rows| {
-                for row in rows {
-                    let (location_id, latitude, longitude, accuracy, timestamp) = row?;
-                    ret += &format!(
-                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
-                        timestamp,
-                        accuracy,
-                        longitude,
-                        latitude
-                    );
-                    location_count += 1;
-                    last_added_location_id = location_id as u32;
-                }
-                Ok(())
-            }
-        )?;
         ret += "</Document>\n</kml>";
     }
 
@@ -462,37 +487,38 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
     )
 }
 
-pub fn set_kml_sent_timestamp(
+pub async fn set_kml_sent_timestamp(
     context: &Context,
     chat_id: ChatId,
     timestamp: i64,
 ) -> Result<(), Error> {
-    sql::execute(
-        context,
-        &context.sql,
-        "UPDATE chats SET locations_last_sent=? WHERE id=?;",
-        params![timestamp, chat_id],
-    )?;
-
+    context
+        .sql
+        .execute(
+            "UPDATE chats SET locations_last_sent=? WHERE id=?;",
+            paramsx![timestamp, chat_id],
+        )
+        .await?;
     Ok(())
 }
 
-pub fn set_msg_location_id(
+pub async fn set_msg_location_id(
     context: &Context,
     msg_id: MsgId,
     location_id: u32,
 ) -> Result<(), Error> {
-    sql::execute(
-        context,
-        &context.sql,
-        "UPDATE msgs SET location_id=? WHERE id=?;",
-        params![location_id, msg_id],
-    )?;
+    context
+        .sql
+        .execute(
+            "UPDATE msgs SET location_id=? WHERE id=?;",
+            paramsx![location_id as i32, msg_id],
+        )
+        .await?;
 
     Ok(())
 }
 
-pub fn save(
+pub async fn save(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
@@ -500,54 +526,69 @@ pub fn save(
     independent: bool,
 ) -> Result<u32, Error> {
     ensure!(!chat_id.is_special(), "Invalid chat id");
-    context
-        .sql
-        .prepare2(
-            "SELECT id FROM locations WHERE timestamp=? AND from_id=?",
-            "INSERT INTO locations\
-             (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent) \
-             VALUES (?,?,?,?,?,?,?);",
-            |mut stmt_test, mut stmt_insert, conn| {
-                let mut newest_timestamp = 0;
-                let mut newest_location_id = 0;
 
-                for location in locations {
-                    let exists =
-                        stmt_test.exists(params![location.timestamp, contact_id as i32])?;
+    let mut newest_timestamp = 0;
+    let mut newest_location_id = 0;
 
-                    if independent || !exists {
-                        stmt_insert.execute(params![
-                            location.timestamp,
-                            contact_id as i32,
-                            chat_id,
-                            location.latitude,
-                            location.longitude,
-                            location.accuracy,
-                            independent,
-                        ])?;
+    for location in locations {
+        let &Location {
+            timestamp,
+            latitude,
+            longitude,
+            accuracy,
+            ..
+        } = location;
 
-                        if location.timestamp > newest_timestamp {
-                            newest_timestamp = location.timestamp;
-                            newest_location_id = sql::get_rowid2_with_conn(
-                                context,
-                                conn,
-                                "locations",
-                                "timestamp",
-                                location.timestamp,
-                                "from_id",
-                                contact_id as i32,
-                            );
-                        }
-                    }
-                }
-                Ok(newest_location_id)
-            },
-        )
-        .map_err(Into::into)
+        let exists: Option<i32> = context
+            .sql
+            .query_value_optional(
+                "SELECT id FROM locations WHERE timestamp=? AND from_id=?",
+                paramsx![timestamp, contact_id as i32],
+            )
+            .await?;
+
+        if independent || exists.is_none() {
+            context
+                .sql
+                .execute(
+                    r#"
+INSERT INTO locations
+    (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent)
+  VALUES (?,?,?,?,?,?,?);
+"#,
+                    paramsx![
+                        timestamp,
+                        contact_id as i32,
+                        chat_id,
+                        latitude,
+                        longitude,
+                        accuracy,
+                        independent
+                    ],
+                )
+                .await?;
+
+            if timestamp > newest_timestamp {
+                // okay to drop, as we use cached prepared statements
+                newest_timestamp = timestamp;
+                newest_location_id = context
+                    .sql
+                    .get_rowid2(
+                        "locations",
+                        "timestamp",
+                        timestamp,
+                        "from_id",
+                        contact_id as i32,
+                    )
+                    .await?;
+            }
+        }
+    }
+
+    Ok(newest_location_id)
 }
 
-#[allow(non_snake_case)]
-pub(crate) fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
+pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
     let now = time();
     let mut continue_streaming = false;
     info!(
@@ -555,101 +596,108 @@ pub(crate) fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Statu
         " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
     );
 
-    if let Ok(rows) = context.sql.query_map(
-        "SELECT id, locations_send_begin, locations_last_sent \
-         FROM chats \
-         WHERE locations_send_until>?;",
-        params![now],
-        |row| {
-            let chat_id: ChatId = row.get(0)?;
-            let locations_send_begin: i64 = row.get(1)?;
-            let locations_last_sent: i64 = row.get(2)?;
-            continue_streaming = true;
+    let pool = match context.sql.get_pool().await {
+        Ok(pool) => pool,
+        Err(err) => {
+            return job::Status::Finished(Err(err.into()));
+        }
+    };
 
-            // be a bit tolerant as the timer may not align exactly with time(NULL)
-            if now - locations_last_sent < (60 - 3) {
-                Ok(None)
-            } else {
-                Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
+    let mut rows = sqlx::query_as(
+        r#"
+SELECT id, locations_send_begin, locations_last_sent
+  FROM chats
+  WHERE locations_send_until>?;
+"#,
+    )
+    .bind(now)
+    .fetch(&pool);
+
+    while let Some(row) = rows.next().await {
+        let (chat_id, locations_send_begin, locations_last_sent): (ChatId, i64, i64) = match row {
+            Ok(res) => res,
+            Err(err) => {
+                warn!(context, "invalid row: {}", err);
+                continue;
             }
-        },
-        |rows| {
-            rows.filter_map(|v| v.transpose())
-                .collect::<Result<Vec<_>, _>>()
-                .map_err(Into::into)
-        },
-    ) {
-        let msgs = context
-            .sql
-            .prepare(
-                "SELECT id \
-                 FROM locations \
-                 WHERE from_id=? \
-                 AND timestamp>=? \
-                 AND timestamp>? \
-                 AND independent=0 \
-                 ORDER BY timestamp;",
-                |mut stmt_locations, _| {
-                    let msgs = rows
-                        .into_iter()
-                        .filter_map(|(chat_id, locations_send_begin, locations_last_sent)| {
-                            if !stmt_locations
-                                .exists(params![
-                                    DC_CONTACT_ID_SELF,
-                                    locations_send_begin,
-                                    locations_last_sent,
-                                ])
-                                .unwrap_or_default()
-                            {
-                                // if there is no new location, there's nothing to send.
-                                // however, maybe we want to bypass this test eg. 15 minutes
-                                None
-                            } else {
-                                // pending locations are attached automatically to every message,
-                                // so also to this empty text message.
-                                // DC_CMD_LOCATION is only needed to create a nicer subject.
-                                //
-                                // for optimisation and to avoid flooding the sending queue,
-                                // we could sending these messages only if we're really online.
-                                // the easiest way to determine this, is to check for an empty message queue.
-                                // (might not be 100%, however, as positions are sent combined later
-                                // and dc_set_location() is typically called periodically, this is ok)
-                                let mut msg = Message::new(Viewtype::Text);
-                                msg.hidden = true;
-                                msg.param.set_cmd(SystemMessage::LocationOnly);
-                                Some((chat_id, msg))
-                            }
-                        })
-                        .collect::<Vec<_>>();
-                    Ok(msgs)
-                },
-            )
-            .unwrap_or_default(); // TODO: Better error handling
+        };
+        continue_streaming = true;
+
+        // be a bit tolerant as the timer may not align exactly with time(NULL)
+        if now - locations_last_sent < (60 - 3) {
+            continue;
+        }
+        let exists = context
+            .sql
+            .exists(
+                r#"
+SELECT id
+  FROM locations
+  WHERE from_id=?
+    AND timestamp>=?
+    AND timestamp>?
+    AND independent=0
+  ORDER BY timestamp;",
+"#,
+                paramsx![
+                    DC_CONTACT_ID_SELF as i32,
+                    locations_send_begin,
+                    locations_last_sent,
+                ],
+            )
+            .await
+            .unwrap_or_default();
+
+        if !exists {
+            // if there is no new location, there's nothing to send.
+            // however, maybe we want to bypass this test eg. 15 minutes
+        } else {
+            // pending locations are attached automatically to every message,
+            // so also to this empty text message.
+            // DC_CMD_LOCATION is only needed to create a nicer subject.
+            //
+            // for optimisation and to avoid flooding the sending queue,
+            // we could sending these messages only if we're really online.
+            // the easiest way to determine this, is to check for an empty message queue.
+            // (might not be 100%, however, as positions are sent combined later
+            // and dc_set_location() is typically called periodically, this is ok)
+            let mut msg = Message::new(Viewtype::Text);
+            msg.hidden = true;
+            msg.param.set_cmd(SystemMessage::LocationOnly);
 
-        for (chat_id, mut msg) in msgs.into_iter() {
             // TODO: better error handling
-            chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
+            chat::send_msg(context, chat_id, &mut msg)
+                .await
+                .unwrap_or_default();
         }
     }
+
     if continue_streaming {
-        schedule_MAYBE_SEND_LOCATIONS(context, true);
+        schedule_maybe_send_locations(context, true).await;
     }
+
     job::Status::Finished(Ok(()))
 }
 
-#[allow(non_snake_case)]
-pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> job::Status {
+pub(crate) async fn job_maybe_send_locations_ended(
+    context: &Context,
+    job: &mut Job,
+) -> job::Status {
     // this function is called when location-streaming _might_ have ended for a chat.
     // the function checks, if location-streaming is really ended;
     // if so, a device-message is added if not yet done.
 
     let chat_id = ChatId::new(job.foreign_id);
 
-    let (send_begin, send_until) = job_try!(context.sql.query_row(
-        "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
-        params![chat_id],
-        |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
-    ));
+    let (send_begin, send_until): (i64, i64) = job_try!(
+        context
+            .sql
+            .query_row(
+                "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
+                paramsx![chat_id],
+            )
+            .await
+    );
 
     if !(send_begin != 0 && time() <= send_until) {
         // still streaming -
@@ -658,13 +706,15 @@ pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> jo
         if !(send_begin == 0 && send_until == 0) {
             // not streaming, device-message already sent
             job_try!(context.sql.execute(
-                "UPDATE chats    SET locations_send_begin=0, locations_send_until=0  WHERE id=?",
-                params![chat_id],
-            ));
+                "UPDATE chats SET locations_send_begin=0, locations_send_until=0  WHERE id=?",
+                paramsx![chat_id],
+            ).await);
 
-            let stock_str = context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-            chat::add_info_msg(context, chat_id, stock_str);
-            context.call_cb(Event::ChatModified(chat_id));
+            let stock_str = context
+                .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
+                .await;
+            chat::add_info_msg(context, chat_id, stock_str).await;
+            context.emit_event(Event::ChatModified(chat_id));
         }
     }
     job::Status::Finished(Ok(()))
@@ -673,11 +723,11 @@ pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> jo
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
-    #[test]
-    fn test_kml_parse() {
-        let context = dummy_context();
+    #[async_std::test]
+    async fn test_kml_parse() {
+        let context = TestContext::new().await;
 
         let xml =
             b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";

src/log.rs

@@ -7,9 +7,7 @@ macro_rules! info {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        let thread = ::std::thread::current();
-        let full = format!("{thid:?} {file}:{line}: {msg}",
-                           thid = thread.id(),
+        let full = format!("{file}:{line}: {msg}",
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
@@ -24,9 +22,7 @@ macro_rules! warn {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        let thread = ::std::thread::current();
-        let full = format!("{thid:?} {file}:{line}: {msg}",
-                           thid = thread.id(),
+        let full = format!("{file}:{line}: {msg}",
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
@@ -48,6 +44,6 @@ macro_rules! error {
 #[macro_export]
 macro_rules! emit_event {
     ($ctx:expr, $event:expr) => {
-        $ctx.call_cb($event);
+        $ctx.emit_event($event);
     };
 }

src/login_param.rs

@@ -50,59 +50,60 @@ impl LoginParam {
     }
 
     /// Read the login parameters from the database.
-    pub fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
+    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
         let key = format!("{}addr", prefix);
         let addr = sql
-            .get_raw_config(context, key)
+            .get_raw_config(key)
+            .await
             .unwrap_or_default()
             .trim()
             .to_string();
 
         let key = format!("{}mail_server", prefix);
-        let mail_server = sql.get_raw_config(context, key).unwrap_or_default();
+        let mail_server = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}mail_port", prefix);
-        let mail_port = sql.get_raw_config_int(context, key).unwrap_or_default();
+        let mail_port = sql.get_raw_config_int(key).await.unwrap_or_default();
 
         let key = format!("{}mail_user", prefix);
-        let mail_user = sql.get_raw_config(context, key).unwrap_or_default();
+        let mail_user = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}mail_pw", prefix);
-        let mail_pw = sql.get_raw_config(context, key).unwrap_or_default();
+        let mail_pw = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}imap_certificate_checks", prefix);
         let imap_certificate_checks =
-            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+            if let Some(certificate_checks) = sql.get_raw_config_int(key).await {
                 num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
             } else {
                 Default::default()
             };
 
         let key = format!("{}send_server", prefix);
-        let send_server = sql.get_raw_config(context, key).unwrap_or_default();
+        let send_server = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}send_port", prefix);
-        let send_port = sql.get_raw_config_int(context, key).unwrap_or_default();
+        let send_port = sql.get_raw_config_int(key).await.unwrap_or_default();
 
         let key = format!("{}send_user", prefix);
-        let send_user = sql.get_raw_config(context, key).unwrap_or_default();
+        let send_user = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}send_pw", prefix);
-        let send_pw = sql.get_raw_config(context, key).unwrap_or_default();
+        let send_pw = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}smtp_certificate_checks", prefix);
         let smtp_certificate_checks =
-            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+            if let Some(certificate_checks) = sql.get_raw_config_int(key).await {
                 num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
             } else {
                 Default::default()
             };
 
         let key = format!("{}server_flags", prefix);
-        let server_flags = sql.get_raw_config_int(context, key).unwrap_or_default();
+        let server_flags = sql.get_raw_config_int(key).await.unwrap_or_default();
 
         LoginParam {
             addr,
@@ -120,12 +121,8 @@ impl LoginParam {
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
-    pub fn save_to_database(
+    pub async fn save_to_database(
         &self,
         context: &Context,
         prefix: impl AsRef<str>,
@@ -134,40 +131,49 @@ impl LoginParam {
         let sql = &context.sql;
 
         let key = format!("{}addr", prefix);
-        sql.set_raw_config(context, key, Some(&self.addr))?;
+        sql.set_raw_config(context, key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_server))?;
+        sql.set_raw_config(context, key, Some(&self.mail_server))
+            .await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(context, key, self.mail_port)?;
+        sql.set_raw_config_int(context, key, self.mail_port).await?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_user))?;
+        sql.set_raw_config(context, key, Some(&self.mail_user))
+            .await?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_pw))?;
+        sql.set_raw_config(context, key, Some(&self.mail_pw))
+            .await?;
 
         let key = format!("{}imap_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)?;
+        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)
+            .await?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_server))?;
+        sql.set_raw_config(context, key, Some(&self.send_server))
+            .await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(context, key, self.send_port)?;
+        sql.set_raw_config_int(context, key, self.send_port).await?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_user))?;
+        sql.set_raw_config(context, key, Some(&self.send_user))
+            .await?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_pw))?;
+        sql.set_raw_config(context, key, Some(&self.send_pw))
+            .await?;
 
         let key = format!("{}smtp_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)?;
+        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)
+            .await?;
 
         let key = format!("{}server_flags", prefix);
-        sql.set_raw_config_int(context, key, self.server_flags)?;
+        sql.set_raw_config_int(context, key, self.server_flags)
+            .await?;
 
         Ok(())
     }
@@ -258,21 +264,15 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-    match certificate_checks {
-        CertificateChecks::Automatic => {
-            // Same as AcceptInvalidCertificates for now.
-            // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
-        }
-        CertificateChecks::Strict => tls_builder,
-        CertificateChecks::AcceptInvalidCertificates
-        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
+
+    if strict_tls {
+        tls_builder
+    } else {
+        tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
+            .danger_accept_invalid_certs(true)
     }
 }
 

src/lot.rs

@@ -1,4 +1,6 @@
-use deltachat_derive::{FromSql, ToSql};
+use deltachat_derive::*;
+
+use crate::key::Fingerprint;
 
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
@@ -14,13 +16,13 @@ pub struct Lot {
     pub(crate) timestamp: i64,
     pub(crate) state: LotState,
     pub(crate) id: u32,
-    pub(crate) fingerprint: Option<String>,
+    pub(crate) fingerprint: Option<Fingerprint>,
     pub(crate) invitenumber: Option<String>,
     pub(crate) auth: Option<String>,
 }
 
 #[repr(u8)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, Sqlx)]
 pub enum Meaning {
     None = 0,
     Text1Draft = 1,
@@ -65,7 +67,7 @@ impl Lot {
 }
 
 #[repr(i32)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, Sqlx)]
 pub enum LotState {
     // Default
     Undefined = 0,

src/mimefactory.rs

@@ -1,3 +1,4 @@
+use async_std::prelude::*;
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
@@ -15,6 +16,7 @@ use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::simplify::escape_message_footer_marks;
 use crate::stock::StockMessage;
 
 // attachments of 25 mb brutto should work on the majority of providers
@@ -65,74 +67,82 @@ pub struct RenderedEmail {
 }
 
 impl<'a, 'b> MimeFactory<'a, 'b> {
-    pub fn from_msg(
+    pub async fn from_msg(
         context: &'a Context,
         msg: &'b Message,
         attach_selfavatar: bool,
     ) -> Result<MimeFactory<'a, 'b>, Error> {
-        let chat = Chat::load_from_db(context, msg.chat_id)?;
+        let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
         let from_addr = context
             .get_config(Config::ConfiguredAddr)
+            .await
+            .unwrap_or_default();
+        let from_displayname = context
+            .get_config(Config::Displayname)
+            .await
             .unwrap_or_default();
-        let from_displayname = context.get_config(Config::Displayname).unwrap_or_default();
         let mut recipients = Vec::with_capacity(5);
         let mut req_mdn = false;
 
         if chat.is_self_talk() {
             recipients.push((from_displayname.to_string(), from_addr.to_string()));
         } else {
-            context.sql.query_map(
-                "SELECT c.authname, c.addr  \
-                 FROM chats_contacts cc  \
-                 LEFT JOIN contacts c ON cc.contact_id=c.id  \
-                 WHERE cc.chat_id=? AND cc.contact_id>9;",
-                params![msg.chat_id],
-                |row| {
-                    let authname: String = row.get(0)?;
-                    let addr: String = row.get(1)?;
-                    Ok((authname, addr))
-                },
-                |rows| {
-                    for row in rows {
-                        let (authname, addr) = row?;
-                        if !recipients_contain_addr(&recipients, &addr) {
-                            recipients.push((authname, addr));
-                        }
-                    }
-                    Ok(())
-                },
-            )?;
+            let pool = context.sql.get_pool().await?;
+
+            let mut rows = sqlx::query_as(
+                r#"
+SELECT c.authname, c.addr
+  FROM chats_contacts cc
+  LEFT JOIN contacts c ON cc.contact_id=c.id
+    WHERE cc.chat_id=? AND cc.contact_id>9;
+"#,
+            )
+            .bind(msg.chat_id)
+            .fetch(&pool);
+
+            while let Some(row) = rows.next().await {
+                let (authname, addr): (String, String) = row?;
+
+                if !recipients_contain_addr(&recipients, &addr) {
+                    recipients.push((authname, addr));
+                }
+            }
 
             let command = msg.param.get_cmd();
 
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
-                && context.get_config_bool(Config::MdnsEnabled)
+                && context.get_config_bool(Config::MdnsEnabled).await
             {
                 req_mdn = true;
             }
         }
-        let (in_reply_to, references) = context.sql.query_row(
-            "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
-            params![msg.id],
-            |row| {
-                let in_reply_to: String = row.get(0)?;
-                let references: String = row.get(1)?;
-
-                Ok((
+        let (in_reply_to, references) = context
+            .sql
+            .query_row(
+                "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
+                paramsx![msg.id],
+            )
+            .await
+            .map(|(in_reply_to, references): (String, String)| {
+                (
                     render_rfc724_mid_list(&in_reply_to),
                     render_rfc724_mid_list(&references),
-                ))
-            },
-        )?;
+                )
+            })?;
 
+        let default_str = context
+            .stock_str(StockMessage::StatusLine)
+            .await
+            .to_string();
         let factory = MimeFactory {
             from_addr,
             from_displayname,
             selfstatus: context
                 .get_config(Config::Selfstatus)
-                .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
+                .await
+                .unwrap_or_else(|| default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -147,29 +157,42 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         Ok(factory)
     }
 
-    pub fn from_mdn(
+    pub async fn from_mdn(
         context: &'a Context,
         msg: &'b Message,
         additional_msg_ids: Vec<String>,
-    ) -> Result<Self, Error> {
+    ) -> Result<MimeFactory<'a, 'b>, Error> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
-        let contact = Contact::load_from_db(context, msg.from_id)?;
+        let contact = Contact::load_from_db(context, msg.from_id).await?;
+        let from_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await
+            .unwrap_or_default();
+        let from_displayname = context
+            .get_config(Config::Displayname)
+            .await
+            .unwrap_or_default();
+        let default_str = context
+            .stock_str(StockMessage::StatusLine)
+            .await
+            .to_string();
+        let selfstatus = context
+            .get_config(Config::Selfstatus)
+            .await
+            .unwrap_or_else(|| default_str);
+        let timestamp = dc_create_smeared_timestamp(context).await;
 
-        Ok(MimeFactory {
+        let res = MimeFactory::<'a, 'b> {
             context,
-            from_addr: context
-                .get_config(Config::ConfiguredAddr)
-                .unwrap_or_default(),
-            from_displayname: context.get_config(Config::Displayname).unwrap_or_default(),
-            selfstatus: context
-                .get_config(Config::Selfstatus)
-                .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
+            from_addr,
+            from_displayname,
+            selfstatus,
             recipients: vec![(
                 contact.get_authname().to_string(),
                 contact.get_addr().to_string(),
             )],
-            timestamp: dc_create_smeared_timestamp(context),
+            timestamp,
             loaded: Loaded::MDN { additional_msg_ids },
             msg,
             in_reply_to: String::default(),
@@ -177,26 +200,31 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             req_mdn: false,
             last_added_location_id: 0,
             attach_selfavatar: false,
-        })
+        };
+
+        Ok(res)
     }
 
-    fn peerstates_for_recipients(&self) -> Result<Vec<(Option<Peerstate>, &str)>, Error> {
+    async fn peerstates_for_recipients(&self) -> Result<Vec<(Option<Peerstate>, &str)>, Error> {
         let self_addr = self
             .context
             .get_config(Config::ConfiguredAddr)
+            .await
             .ok_or_else(|| format_err!("Not configured"))?;
 
-        Ok(self
+        let mut res = Vec::new();
+        for (_, addr) in self
             .recipients
             .iter()
             .filter(|(_, addr)| addr != &self_addr)
-            .map(|(_, addr)| {
-                (
-                    Peerstate::from_addr(self.context, &self.context.sql, addr),
-                    addr.as_str(),
-                )
-            })
-            .collect())
+        {
+            res.push((
+                Peerstate::from_addr(self.context, addr).await.ok(),
+                addr.as_str(),
+            ));
+        }
+
+        Ok(res)
     }
 
     fn is_e2ee_guaranteed(&self) -> bool {
@@ -256,11 +284,11 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn should_do_gossip(&self) -> bool {
+    async fn should_do_gossip(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
-                let gossiped_timestamp = chat.get_gossiped_timestamp(self.context);
+                let gossiped_timestamp = chat.get_gossiped_timestamp(self.context).await;
                 if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
                     return true;
                 }
@@ -301,12 +329,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn subject_str(&self) -> String {
+    async fn subject_str(&self) -> String {
         match self.loaded {
             Loaded::Message { ref chat } => {
                 if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
                     self.context
                         .stock_str(StockMessage::AcSetupMsgSubject)
+                        .await
                         .into_owned()
                 } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
                     let re = if self.in_reply_to.is_empty() {
@@ -316,18 +345,54 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    let raw = message::get_summarytext_by_raw(
-                        self.msg.viewtype,
-                        self.msg.text.as_ref(),
-                        &self.msg.param,
-                        32,
-                        self.context,
-                    );
-                    let raw_subject = raw.lines().next().unwrap_or_default();
-                    format!("Chat: {}", raw_subject)
+                    match chat.param.get(Param::LastSubject) {
+                        Some(last_subject) => {
+                            let subject_start = if last_subject.starts_with("Chat:") {
+                                0
+                            } else {
+                                // "Antw:" is the longest abbreviation in
+                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+                                // so look at the first _5_ characters:
+                                match last_subject.chars().take(5).position(|c| c == ':') {
+                                    Some(prefix_end) => prefix_end + 1,
+                                    None => 0,
+                                }
+                            };
+                            format!(
+                                "Re: {}",
+                                last_subject
+                                    .chars()
+                                    .skip(subject_start)
+                                    .collect::<String>()
+                                    .trim()
+                            )
+                        }
+                        None => {
+                            let self_name = match self.context.get_config(Config::Displayname).await
+                            {
+                                Some(name) => name,
+                                None => self
+                                    .context
+                                    .get_config(Config::Addr)
+                                    .await
+                                    .unwrap_or_default(),
+                            };
+
+                            self.context
+                                .stock_string_repl_str(
+                                    StockMessage::SubjectForNewContact,
+                                    self_name,
+                                )
+                                .await
+                        }
+                    }
                 }
             }
-            Loaded::MDN { .. } => self.context.stock_str(StockMessage::ReadRcpt).into_owned(),
+            Loaded::MDN { .. } => self
+                .context
+                .stock_str(StockMessage::ReadRcpt)
+                .await
+                .into_owned(),
         }
     }
 
@@ -338,7 +403,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .collect()
     }
 
-    pub fn render(mut self) -> Result<RenderedEmail, Error> {
+    pub async fn render(mut self) -> Result<RenderedEmail, Error> {
         // Headers that are encrypted
         // - Chat-*, except Chat-Version
         // - Secure-Join*
@@ -369,6 +434,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
+        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -421,17 +488,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let min_verified = self.min_verified();
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
-        let subject_str = self.subject_str();
+        let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
-        let mut encrypt_helper = EncryptHelper::new(self.context)?;
+        let encrypt_helper = EncryptHelper::new(self.context).await?;
 
         let subject = encode_words(&subject_str);
 
         let mut message = match self.loaded {
             Loaded::Message { .. } => {
-                self.render_message(&mut protected_headers, &mut unprotected_headers, &grpimage)?
+                self.render_message(&mut protected_headers, &mut unprotected_headers, &grpimage)
+                    .await?
             }
-            Loaded::MDN { .. } => self.render_mdn()?,
+            Loaded::MDN { .. } => self.render_mdn().await?,
         };
 
         if force_plaintext != ForcePlaintext::NoAutocryptHeader as i32 {
@@ -442,7 +510,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         protected_headers.push(Header::new("Subject".into(), subject));
 
-        let peerstates = self.peerstates_for_recipients()?;
+        let peerstates = self.peerstates_for_recipients().await?;
         let should_encrypt =
             encrypt_helper.should_encrypt(self.context, e2ee_guaranteed, &peerstates)?;
         let is_encrypted = should_encrypt && force_plaintext == 0;
@@ -470,7 +538,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         let outer_message = if is_encrypted {
             // Add gossip headers in chats with multiple recipients
-            if peerstates.len() > 1 && self.should_do_gossip() {
+            if peerstates.len() > 1 && self.should_do_gossip().await {
                 for peerstate in peerstates.iter().filter_map(|(state, _)| state.as_ref()) {
                     if peerstate.peek_key(min_verified).is_some() {
                         if let Some(header) = peerstate.render_gossip_header(min_verified) {
@@ -518,8 +586,9 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 println!("{}", raw_message);
             }
 
-            let encrypted =
-                encrypt_helper.encrypt(self.context, min_verified, message, &peerstates)?;
+            let encrypted = encrypt_helper
+                .encrypt(self.context, min_verified, message, peerstates)
+                .await?;
 
             outer_message = outer_message
                 .child(
@@ -591,9 +660,9 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         Some(part)
     }
 
-    fn get_location_kml_part(&mut self) -> Result<PartBuilder, Error> {
+    async fn get_location_kml_part(&mut self) -> Result<PartBuilder, Error> {
         let (kml_content, last_added_location_id) =
-            location::get_kml(self.context, self.msg.chat_id)?;
+            location::get_kml(self.context, self.msg.chat_id).await?;
         let part = PartBuilder::new()
             .content_type(
                 &"application/vnd.google-earth.kml+xml"
@@ -613,7 +682,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     }
 
     #[allow(clippy::cognitive_complexity)]
-    fn render_message(
+    async fn render_message(
         &mut self,
         protected_headers: &mut Vec<Header>,
         unprotected_headers: &mut Vec<Header>,
@@ -708,6 +777,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 placeholdertext = Some(
                     self.context
                         .stock_str(StockMessage::AcSetupMsgBody)
+                        .await
                         .to_string(),
                 );
             }
@@ -754,7 +824,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             meta.viewtype = Viewtype::Image;
             meta.param.set(Param::File, grpimage);
 
-            let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image")?;
+            let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
             meta_part = Some(mail);
             protected_headers.push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
         }
@@ -807,7 +877,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let message_text = format!(
             "{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            &final_text,
+            escape_message_footer_marks(final_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -825,13 +895,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         // add attachment part
         if chat::msgtype_has_file(self.msg.viewtype) {
-            if !is_file_size_okay(context, &self.msg) {
+            if !is_file_size_okay(context, &self.msg).await {
                 bail!(
                     "Message exceeds the recommended {} MB.",
                     RECOMMENDED_FILE_SIZE / 1_000_000,
                 );
             } else {
-                let (file_part, _) = build_body_file(context, &self.msg, "")?;
+                let (file_part, _) = build_body_file(context, &self.msg, "").await?;
                 parts.push(file_part);
             }
         }
@@ -844,8 +914,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             parts.push(msg_kml_part);
         }
 
-        if location::is_sending_locations_to_chat(context, self.msg.chat_id) {
-            match self.get_location_kml_part() {
+        if location::is_sending_locations_to_chat(context, self.msg.chat_id).await {
+            match self.get_location_kml_part().await {
                 Ok(part) => parts.push(part),
                 Err(err) => {
                     warn!(context, "mimefactory: could not send location: {}", err);
@@ -854,7 +924,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
 
         if self.attach_selfavatar {
-            match context.get_config(Config::Selfavatar) {
+            match context.get_config(Config::Selfavatar).await {
                 Some(path) => match build_selfavatar_file(context, &path) {
                     Ok((part, filename)) => {
                         parts.push(part);
@@ -881,7 +951,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     }
 
     /// Render an MDN
-    fn render_mdn(&mut self) -> Result<PartBuilder, Error> {
+    async fn render_mdn(&mut self) -> Result<PartBuilder, Error> {
         // RFC 6522, this also requires the `report-type` parameter which is equal
         // to the MIME subtype of the second body part of the multipart/report
         //
@@ -916,13 +986,15 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         {
             self.context
                 .stock_str(StockMessage::EncryptedMsg)
+                .await
                 .into_owned()
         } else {
-            self.msg.get_summarytext(self.context, 32)
+            self.msg.get_summarytext(self.context, 32).await
         };
         let p2 = self
             .context
-            .stock_string_repl_str(StockMessage::ReadRcptMailBody, p1);
+            .stock_string_repl_str(StockMessage::ReadRcptMailBody, p1)
+            .await;
         let message_text = format!("{}\r\n", p2);
         message = message.child(
             PartBuilder::new()
@@ -979,14 +1051,15 @@ fn wrapped_base64_encode(buf: &[u8]) -> String {
         .join("\r\n")
 }
 
-fn build_body_file(
+async fn build_body_file(
     context: &Context,
     msg: &Message,
     base_name: &str,
 ) -> Result<(PartBuilder, String), Error> {
     let blob = msg
         .param
-        .get_blob(Param::File, context, true)?
+        .get_blob(Param::File, context, true)
+        .await?
         .ok_or_else(|| format_err!("msg has no filename"))?;
     let suffix = blob.suffix().unwrap_or("dat");
 
@@ -1082,10 +1155,10 @@ fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool
         .any(|(_, cur)| cur.to_lowercase() == addr_lc)
 }
 
-fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
+async fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
     match msg.param.get_path(Param::File, context).unwrap_or(None) {
         Some(path) => {
-            let bytes = dc_get_filebytes(context, &path);
+            let bytes = dc_get_filebytes(context, &path).await;
             bytes <= UPPER_LIMIT_FILE_SIZE
         }
         None => false,
@@ -1128,6 +1201,10 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chatlist::Chatlist;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::mimeparser::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1135,6 +1212,9 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
+        assert!(!display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1146,6 +1226,25 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
+    #[test]
+    fn test_render_email_address_noescape() {
+        let display_name = "a space";
+        let addr = "x@y.org";
+
+        assert!(display_name.is_ascii());
+        assert!(display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+        let s = format!(
+            "{}",
+            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
+        );
+
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        assert_eq!(s, "a space <x@y.org>");
+    }
+
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1188,4 +1287,193 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
+
+    #[async_std::test]
+    async fn test_subject() {
+        // 1.: Receive a mail from an MUA or Delta Chat
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Infos: 42"
+        );
+
+        // 2. Receive a message from Delta Chat when we did not send any messages before
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        // 3. Send the first message to a new contact
+        let t = TestContext::new_alice().await;
+
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::Displayname, Some("Alice"))
+            .await
+            .unwrap();
+        assert_eq!(first_subject_str(t).await, "Message from Alice");
+
+        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+    }
+
+    async fn first_subject_str(t: TestContext) -> String {
+        let contact_id =
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.com", Origin::ManuallyCreated)
+                .await
+                .unwrap()
+                .0;
+
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+
+        mf.subject_str().await
+    }
+
+    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+        let t = TestContext::new_alice().await;
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        mf.subject_str().await
+    }
+
+    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+        context
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(context, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        new_msg
+    }
+
+    #[async_std::test]
+    // This test could still be extended
+    async fn test_render_reply() {
+        let t = TestContext::new_alice().await;
+        let context = &t.ctx;
+
+        let msg = incoming_msg_to_reply_msg(
+            b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            context,
+        )
+        .await;
+
+        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+
+        let recipients = mimefactory.recipients();
+        assert_eq!(recipients, vec!["charlie@example.com"]);
+
+        let rendered_msg = mimefactory.render().await.unwrap();
+
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        assert_eq!(
+            mail.headers
+                .iter()
+                .find(|h| h.get_key() == "MIME-Version")
+                .unwrap()
+                .get_value(),
+            "1.0"
+        );
+
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+            .await
+            .unwrap();
+    }
 }

src/oauth2.rs

@@ -1,12 +1,16 @@
 //! OAuth 2 module
 
+use regex::Regex;
 use std::collections::HashMap;
 
+use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::provider;
+use crate::provider::Oauth2Authorizer;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
@@ -15,6 +19,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
+    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -24,8 +29,11 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
+    mx_pattern: None,
 };
 
+const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -33,6 +41,7 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
+    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -48,12 +57,12 @@ struct Response {
     scope: Option<String>,
 }
 
-pub fn dc_get_oauth2_url(
+pub async fn dc_get_oauth2_url(
     context: &Context,
     addr: impl AsRef<str>,
     redirect_uri: impl AsRef<str>,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
             .set_raw_config(
@@ -61,6 +70,7 @@ pub fn dc_get_oauth2_url(
                 "oauth2_pending_redirect_uri",
                 Some(redirect_uri.as_ref()),
             )
+            .await
             .is_err()
         {
             return None;
@@ -74,21 +84,18 @@ pub fn dc_get_oauth2_url(
     }
 }
 
-// The following function may block due http-requests;
-// must not be called from the main thread or by the ui!
-pub fn dc_get_oauth2_access_token(
+pub async fn dc_get_oauth2_access_token(
     context: &Context,
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
     regenerate: bool,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
-        let lock = context.oauth2_critical.clone();
-        let _l = lock.lock().unwrap();
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
+        let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
-        if !regenerate && !is_expired(context) {
-            let access_token = context.sql.get_raw_config(context, "oauth2_access_token");
+        if !regenerate && !is_expired(context).await {
+            let access_token = context.sql.get_raw_config("oauth2_access_token").await;
             if access_token.is_some() {
                 // success
                 return access_token;
@@ -96,10 +103,11 @@ pub fn dc_get_oauth2_access_token(
         }
 
         // generate new token: build & call auth url
-        let refresh_token = context.sql.get_raw_config(context, "oauth2_refresh_token");
+        let refresh_token = context.sql.get_raw_config("oauth2_refresh_token").await;
         let refresh_token_for = context
             .sql
-            .get_raw_config(context, "oauth2_refresh_token_for")
+            .get_raw_config("oauth2_refresh_token_for")
+            .await
             .unwrap_or_else(|| "unset".into());
 
         let (redirect_uri, token_url, update_redirect_uri_on_success) =
@@ -108,7 +116,8 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_raw_config(context, "oauth2_pending_redirect_uri")
+                        .get_raw_config("oauth2_pending_redirect_uri")
+                        .await
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.init_token,
                     true,
@@ -121,7 +130,8 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_raw_config(context, "oauth2_redirect_uri")
+                        .get_raw_config("oauth2_redirect_uri")
+                        .await
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.refresh_token,
                     false,
@@ -154,10 +164,7 @@ pub fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = reqwest::blocking::Client::new()
-            .post(post_url)
-            .form(&post_param)
-            .send();
+        let response = surf::post(post_url).body_form(&post_param);
         if response.is_err() {
             warn!(
                 context,
@@ -165,19 +172,8 @@ pub fn dc_get_oauth2_access_token(
             );
             return None;
         }
-        let response = response.unwrap();
-        if !response.status().is_success() {
-            warn!(
-                context,
-                "Unsuccessful response when calling OAuth2 at {}: {:?}",
-                token_url,
-                response.status()
-            );
-            return None;
-        }
 
-        // generate new token: parse returned json
-        let parsed: reqwest::Result<Response> = response.json();
+        let parsed: Result<Response, _> = response.unwrap().recv_json().await;
         if parsed.is_err() {
             warn!(
                 context,
@@ -185,7 +181,6 @@ pub fn dc_get_oauth2_access_token(
             );
             return None;
         }
-        println!("response: {:?}", &parsed);
 
         // update refresh_token if given, typically on the first round, but we update it later as well.
         let response = parsed.unwrap();
@@ -193,10 +188,12 @@ pub fn dc_get_oauth2_access_token(
             context
                 .sql
                 .set_raw_config(context, "oauth2_refresh_token", Some(token))
+                .await
                 .ok();
             context
                 .sql
                 .set_raw_config(context, "oauth2_refresh_token_for", Some(code.as_ref()))
+                .await
                 .ok();
         }
 
@@ -206,6 +203,7 @@ pub fn dc_get_oauth2_access_token(
             context
                 .sql
                 .set_raw_config(context, "oauth2_access_token", Some(token))
+                .await
                 .ok();
             let expires_in = response
                 .expires_in
@@ -215,18 +213,22 @@ pub fn dc_get_oauth2_access_token(
             context
                 .sql
                 .set_raw_config_int64(context, "oauth2_timestamp_expires", expires_in)
+                .await
                 .ok();
 
             if update_redirect_uri_on_success {
                 context
                     .sql
                     .set_raw_config(context, "oauth2_redirect_uri", Some(redirect_uri.as_ref()))
+                    .await
                     .ok();
             }
         } else {
             warn!(context, "Failed to find OAuth2 access token");
         }
 
+        drop(lock);
+
         response.access_token
     } else {
         warn!(context, "Internal OAuth2 error: 2");
@@ -235,22 +237,23 @@ pub fn dc_get_oauth2_access_token(
     }
 }
 
-pub fn dc_get_oauth2_addr(
+pub async fn dc_get_oauth2_addr(
     context: &Context,
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    let oauth2 = Oauth2::from_address(addr.as_ref()).await?;
     oauth2.get_userinfo?;
 
     if let Some(access_token) =
-        dc_get_oauth2_access_token(context, addr.as_ref(), code.as_ref(), false)
+        dc_get_oauth2_access_token(context, addr.as_ref(), code.as_ref(), false).await
     {
-        let addr_out = oauth2.get_addr(context, access_token);
+        let addr_out = oauth2.get_addr(context, access_token).await;
         if addr_out.is_none() {
             // regenerate
-            if let Some(access_token) = dc_get_oauth2_access_token(context, addr, code, true) {
-                oauth2.get_addr(context, access_token)
+            if let Some(access_token) = dc_get_oauth2_access_token(context, addr, code, true).await
+            {
+                oauth2.get_addr(context, access_token).await
             } else {
                 None
             }
@@ -263,24 +266,57 @@ pub fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    fn from_address(addr: impl AsRef<str>) -> Option<Self> {
+    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
         let addr_normalized = normalize_addr(addr.as_ref());
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            match domain {
-                "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
-                | "narod.ru" => Some(OAUTH2_YANDEX),
-                _ => None,
+            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
+                match &provider.oauth2_authorizer {
+                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
+                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
+                    None => None, // provider known to not support oauth2, no mx-lookup required
+                }
+            } else {
+                Oauth2::lookup_mx(domain).await
             }
         } else {
             None
         }
     }
 
-    fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
+    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
+        if let Ok(resolver) = resolver(
+            config::ResolverConfig::default(),
+            config::ResolverOpts::default(),
+        )
+        .await
+        {
+            for provider in OAUTH2_PROVIDERS.iter() {
+                if let Some(pattern) = provider.mx_pattern {
+                    let re = Regex::new(pattern).unwrap();
+
+                    let mut fqdn: String = String::from(domain.as_ref());
+                    if !fqdn.ends_with('.') {
+                        fqdn.push_str(".");
+                    }
+
+                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
+                        for rr in res.iter() {
+                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
+                                return Some(provider.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
+    async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
         let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
 
@@ -291,50 +327,35 @@ impl Oauth2 {
         //   "verified_email": true,
         //   "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg"
         // }
-        let response = reqwest::blocking::Client::new().get(&userinfo_url).send();
+        let response: Result<HashMap<String, serde_json::Value>, surf::Error> =
+            surf::get(userinfo_url).recv_json().await;
         if response.is_err() {
             warn!(context, "Error getting userinfo: {:?}", response);
             return None;
         }
-        let response = response.unwrap();
-        if !response.status().is_success() {
-            warn!(context, "Error getting userinfo: {:?}", response.status());
-            return None;
-        }
 
-        let parsed: reqwest::Result<HashMap<String, serde_json::Value>> = response.json();
-        if parsed.is_err() {
-            warn!(
-                context,
-                "Failed to parse userinfo JSON response: {:?}", parsed
-            );
-            return None;
-        }
-        if let Ok(response) = parsed {
-            // CAVE: serde_json::Value.as_str() removes the quotes of json-strings
-            // but serde_json::Value.to_string() does not!
-            if let Some(addr) = response.get("email") {
-                if let Some(s) = addr.as_str() {
-                    Some(s.to_string())
-                } else {
-                    warn!(context, "E-mail in userinfo is not a string: {}", addr);
-                    None
-                }
+        let parsed = response.unwrap();
+        // CAVE: serde_json::Value.as_str() removes the quotes of json-strings
+        // but serde_json::Value.to_string() does not!
+        if let Some(addr) = parsed.get("email") {
+            if let Some(s) = addr.as_str() {
+                Some(s.to_string())
             } else {
-                warn!(context, "E-mail missing in userinfo.");
+                warn!(context, "E-mail in userinfo is not a string: {}", addr);
                 None
             }
         } else {
-            warn!(context, "Failed to parse userinfo.");
+            warn!(context, "E-mail missing in userinfo.");
             None
         }
     }
 }
 
-fn is_expired(context: &Context) -> bool {
+async fn is_expired(context: &Context) -> bool {
     let expire_timestamp = context
         .sql
-        .get_raw_config_int64(context, "oauth2_timestamp_expires")
+        .get_raw_config_int64("oauth2_timestamp_expires")
+        .await
         .unwrap_or_default();
 
     if expire_timestamp <= 0 {
@@ -377,48 +398,62 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_oauth_from_address() {
-        assert_eq!(Oauth2::from_address("hello@gmail.com"), Some(OAUTH2_GMAIL));
+    #[async_std::test]
+    async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com"),
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com"),
+            Oauth2::from_address("hello@googlemail.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.com").await,
+            Some(OAUTH2_YANDEX)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address("hello@yandex.ru"), Some(OAUTH2_YANDEX));
 
-        assert_eq!(Oauth2::from_address("hello@web.de"), None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
     }
 
-    #[test]
-    fn test_dc_get_oauth2_addr() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_oauth_from_mx() {
+        assert_eq!(
+            Oauth2::from_address("hello@google.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+    }
+
+    #[async_std::test]
+    async fn test_dc_get_oauth2_addr() {
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
-        let res = dc_get_oauth2_addr(&ctx.ctx, addr, code);
+        let res = dc_get_oauth2_addr(&ctx.ctx, addr, code).await;
         // this should fail as it is an invalid password
         assert_eq!(res, None);
     }
 
-    #[test]
-    fn test_dc_get_oauth2_url() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_dc_get_oauth2_url() {
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
-        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri);
+        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
 
         assert_eq!(res, Some("https://accounts.google.com/o/oauth2/auth?client_id=959970109878%2D4mvtgf6feshskf7695nfln6002mom908%2Eapps%2Egoogleusercontent%2Ecom&redirect_uri=chat%2Edelta%3A%2Fcom%2Eb44t%2Emessenger&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline".into()));
     }
 
-    #[test]
-    fn test_dc_get_oauth2_token() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_dc_get_oauth2_token() {
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
-        let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false);
+        let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false).await;
         // this should fail as it is an invalid password
         assert_eq!(res, None);
     }

src/param.rs

@@ -1,8 +1,8 @@
 use std::collections::BTreeMap;
 use std::fmt;
-use std::path::PathBuf;
 use std::str;
 
+use async_std::path::PathBuf;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
@@ -65,9 +65,6 @@ pub enum Param {
     /// For Messages
     Arg4 = b'H',
 
-    /// For Messages
-    Error = b'L',
-
     /// For Messages
     AttachGroupImage = b'A',
 
@@ -103,6 +100,9 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
+    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    LastSubject = b't',
+
     /// For Chats
     Devicetalk = b'D',
 
@@ -117,8 +117,6 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
-
-    MailingList = b't',
 }
 
 /// Possible values for `Param::ForcePlaintext`.
@@ -277,7 +275,8 @@ impl Params {
     /// created without copying if the path already referes to a valid
     /// blob.  If so a [BlobObject] will be returned regardless of the
     /// `create` argument.
-    pub fn get_blob<'a>(
+    #[allow(clippy::needless_lifetimes)]
+    pub async fn get_blob<'a>(
         &self,
         key: Param,
         context: &'a Context,
@@ -290,7 +289,7 @@ impl Params {
         let file = ParamsFile::from_param(context, val)?;
         let blob = match file {
             ParamsFile::FsPath(path) => match create {
-                true => BlobObject::new_from_path(context, path)?,
+                true => BlobObject::new_from_path(context, path).await?,
                 false => BlobObject::from_path(context, path)?,
             },
             ParamsFile::Blob(blob) => blob,
@@ -364,8 +363,8 @@ impl<'a> ParamsFile<'a> {
 mod tests {
     use super::*;
 
-    use std::fs;
-    use std::path::Path;
+    use async_std::fs;
+    use async_std::path::Path;
 
     use crate::test_utils::*;
 
@@ -413,9 +412,9 @@ mod tests {
         assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
     }
 
-    #[test]
-    fn test_params_file_fs_path() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_params_file_fs_path() {
+        let t = TestContext::new().await;
         if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
@@ -423,9 +422,9 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_params_file_blob() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_params_file_blob() {
+        let t = TestContext::new().await;
         if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
@@ -434,28 +433,33 @@ mod tests {
     }
 
     // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
-    #[test]
-    fn test_params_get_fileparam() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_params_get_fileparam() {
+        let t = TestContext::new().await;
         let fname = t.dir.path().join("foo");
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());
 
         let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
-        assert_eq!(file, ParamsFile::FsPath(fname.clone()));
+        assert_eq!(file, ParamsFile::FsPath(fname.clone().into()));
 
-        let path = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        let path: PathBuf = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        let fname: PathBuf = fname.into();
         assert_eq!(path, fname);
 
         // Blob does not exist yet, expect BlobError.
-        let err = p.get_blob(Param::File, &t.ctx, false).unwrap_err();
+        let err = p.get_blob(Param::File, &t.ctx, false).await.unwrap_err();
         match err {
             BlobError::WrongBlobdir { .. } => (),
             _ => panic!("wrong error type/variant: {:?}", err),
         }
 
-        fs::write(fname, b"boo").unwrap();
-        let blob = p.get_blob(Param::File, &t.ctx, true).unwrap().unwrap();
+        fs::write(fname, b"boo").await.unwrap();
+        let blob = p
+            .get_blob(Param::File, &t.ctx, true)
+            .await
+            .unwrap()
+            .unwrap();
         assert_eq!(
             blob,
             BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
@@ -464,7 +468,11 @@ mod tests {
         // Blob in blobdir, expect blob.
         let bar = t.ctx.get_blobdir().join("bar");
         p.set(Param::File, bar.to_str().unwrap());
-        let blob = p.get_blob(Param::File, &t.ctx, false).unwrap().unwrap();
+        let blob = p
+            .get_blob(Param::File, &t.ctx, false)
+            .await
+            .unwrap()
+            .unwrap();
         assert_eq!(
             blob,
             BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
@@ -473,6 +481,10 @@ mod tests {
         p.remove(Param::File);
         assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
         assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p.get_blob(Param::File, &t.ctx, false).unwrap().is_none());
+        assert!(p
+            .get_blob(Param::File, &t.ctx, false)
+            .await
+            .unwrap()
+            .is_none());
     }
 }

src/peerstate.rs

@@ -1,15 +1,13 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
-use std::collections::HashSet;
-use std::convert::TryFrom;
-use std::fmt;
 
-use num_traits::FromPrimitive;
+use std::collections::HashSet;
+
+use anyhow::Result;
 
 use crate::aheader::*;
-use crate::constants::*;
 use crate::context::Context;
-use crate::key::{Key, SignedPublicKey};
-use crate::sql::{self, Sql};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
+use crate::sql::Sql;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -26,60 +24,66 @@ pub enum PeerstateVerifiedStatus {
 }
 
 /// Peerstate represents the state of an Autocrypt peer.
-pub struct Peerstate<'a> {
-    pub context: &'a Context,
+#[derive(Debug, PartialEq, Eq)]
+pub struct Peerstate {
     pub addr: String,
     pub last_seen: i64,
     pub last_seen_autocrypt: i64,
     pub prefer_encrypt: EncryptPreference,
-    pub public_key: Option<Key>,
-    pub public_key_fingerprint: Option<String>,
-    pub gossip_key: Option<Key>,
+    pub public_key: Option<SignedPublicKey>,
+    pub public_key_fingerprint: Option<Fingerprint>,
+    pub gossip_key: Option<SignedPublicKey>,
     pub gossip_timestamp: i64,
-    pub gossip_key_fingerprint: Option<String>,
-    pub verified_key: Option<Key>,
-    pub verified_key_fingerprint: Option<String>,
+    pub gossip_key_fingerprint: Option<Fingerprint>,
+    pub verified_key: Option<SignedPublicKey>,
+    pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
     pub degrade_event: Option<DegradeEvent>,
 }
 
-impl<'a> PartialEq for Peerstate<'a> {
-    fn eq(&self, other: &Peerstate) -> bool {
-        self.addr == other.addr
-            && self.last_seen == other.last_seen
-            && self.last_seen_autocrypt == other.last_seen_autocrypt
-            && self.prefer_encrypt == other.prefer_encrypt
-            && self.public_key == other.public_key
-            && self.public_key_fingerprint == other.public_key_fingerprint
-            && self.gossip_key == other.gossip_key
-            && self.gossip_timestamp == other.gossip_timestamp
-            && self.gossip_key_fingerprint == other.gossip_key_fingerprint
-            && self.verified_key == other.verified_key
-            && self.verified_key_fingerprint == other.verified_key_fingerprint
-            && self.to_save == other.to_save
-            && self.degrade_event == other.degrade_event
-    }
-}
+impl<'a> sqlx::FromRow<'a, sqlx::sqlite::SqliteRow> for Peerstate {
+    fn from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Self, sqlx::Error> {
+        use sqlx::Row;
 
-impl<'a> Eq for Peerstate<'a> {}
+        let mut res = Self::new(row.try_get("addr")?);
 
-impl<'a> fmt::Debug for Peerstate<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.debug_struct("Peerstate")
-            .field("addr", &self.addr)
-            .field("last_seen", &self.last_seen)
-            .field("last_seen_autocrypt", &self.last_seen_autocrypt)
-            .field("prefer_encrypt", &self.prefer_encrypt)
-            .field("public_key", &self.public_key)
-            .field("public_key_fingerprint", &self.public_key_fingerprint)
-            .field("gossip_key", &self.gossip_key)
-            .field("gossip_timestamp", &self.gossip_timestamp)
-            .field("gossip_key_fingerprint", &self.gossip_key_fingerprint)
-            .field("verified_key", &self.verified_key)
-            .field("verified_key_fingerprint", &self.verified_key_fingerprint)
-            .field("to_save", &self.to_save)
-            .field("degrade_event", &self.degrade_event)
-            .finish()
+        res.last_seen = row.try_get_unchecked("last_seen")?;
+        res.last_seen_autocrypt = row.try_get_unchecked("last_seen_autocrypt")?;
+        res.prefer_encrypt = row.try_get("prefer_encrypted")?;
+        res.gossip_timestamp = row.try_get_unchecked("gossip_timestamp")?;
+
+        res.public_key_fingerprint = row
+            .try_get::<Option<String>, _>("public_key_fingerprint")?
+            .map(|fp| fp.parse::<Fingerprint>())
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.gossip_key_fingerprint = row
+            .try_get::<Option<String>, _>("gossip_key_fingerprint")?
+            .map(|fp| fp.parse::<Fingerprint>())
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.verified_key_fingerprint = row
+            .try_get::<Option<String>, _>("verified_key_fingerprint")?
+            .map(|fp| fp.parse::<Fingerprint>())
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.public_key = row
+            .try_get::<Option<&[u8]>, _>("public_key")?
+            .map(|blob| SignedPublicKey::from_slice(blob))
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.gossip_key = row
+            .try_get::<Option<&[u8]>, _>("gossip_key")?
+            .map(|blob| SignedPublicKey::from_slice(blob))
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.verified_key = row
+            .try_get::<Option<&[u8]>, _>("verified_key")?
+            .map(|blob| SignedPublicKey::from_slice(blob))
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+
+        Ok(res)
     }
 }
 
@@ -100,10 +104,9 @@ pub enum DegradeEvent {
     FingerprintChanged = 0x02,
 }
 
-impl<'a> Peerstate<'a> {
-    pub fn new(context: &'a Context, addr: String) -> Self {
+impl Peerstate {
+    pub fn new(addr: String) -> Self {
         Peerstate {
-            context,
             addr,
             last_seen: 0,
             last_seen_autocrypt: 0,
@@ -120,114 +123,77 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, header.addr.clone());
+    pub fn from_header(header: &Aheader, message_time: i64) -> Self {
+        let mut res = Self::new(header.addr.clone());
 
         res.last_seen = message_time;
         res.last_seen_autocrypt = message_time;
         res.to_save = Some(ToSave::All);
         res.prefer_encrypt = header.prefer_encrypt;
-        res.public_key = Some(Key::from(header.public_key.clone()));
+        res.public_key = Some(header.public_key.clone());
         res.recalc_fingerprint();
 
         res
     }
 
-    pub fn from_gossip(context: &'a Context, gossip_header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, gossip_header.addr.clone());
+    pub fn from_gossip(gossip_header: &Aheader, message_time: i64) -> Self {
+        let mut res = Self::new(gossip_header.addr.clone());
 
         res.gossip_timestamp = message_time;
         res.to_save = Some(ToSave::All);
-        res.gossip_key = Some(Key::from(gossip_header.public_key.clone()));
+        res.gossip_key = Some(gossip_header.public_key.clone());
         res.recalc_fingerprint();
 
         res
     }
 
-    pub fn from_addr(context: &'a Context, _sql: &Sql, addr: &str) -> Option<Self> {
-        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, verified_key, verified_key_fingerprint FROM acpeerstates  WHERE addr=? COLLATE NOCASE;";
-        Self::from_stmt(context, query, &[addr])
+    pub async fn from_addr(context: &Context, addr: &str) -> Result<Peerstate> {
+        let query = r#"
+SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, 
+       gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, 
+       verified_key, verified_key_fingerprint 
+  FROM acpeerstates  
+  WHERE addr=? COLLATE NOCASE;
+"#;
+        Self::from_stmt(context, query, paramsx![addr]).await
     }
 
-    pub fn from_fingerprint(context: &'a Context, _sql: &Sql, fingerprint: &str) -> Option<Self> {
-        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
-                     gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint \
-                     FROM acpeerstates  \
-                     WHERE public_key_fingerprint=? COLLATE NOCASE \
-                     OR gossip_key_fingerprint=? COLLATE NOCASE  \
-                     ORDER BY public_key_fingerprint=? DESC;";
+    pub async fn from_fingerprint(
+        context: &Context,
+        fingerprint: &Fingerprint,
+    ) -> Result<Peerstate> {
+        let query = r#"
+SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key,
+       gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint,
+       verified_key, verified_key_fingerprint
+  FROM acpeerstates
+  WHERE public_key_fingerprint=? COLLATE NOCASE
+    OR gossip_key_fingerprint=? COLLATE NOCASE
+  ORDER BY public_key_fingerprint=? DESC;
+"#;
 
+        let fingerprint = fingerprint.hex();
         Self::from_stmt(
             context,
             query,
-            params![fingerprint, fingerprint, fingerprint],
+            paramsx![&fingerprint, &fingerprint, &fingerprint],
         )
+        .await
     }
 
-    fn from_stmt<P>(context: &'a Context, query: &str, params: P) -> Option<Self>
-    where
-        P: IntoIterator,
-        P::Item: rusqlite::ToSql,
-    {
-        context
-            .sql
-            .query_row(query, params, |row| {
-                /* all the above queries start with this: SELECT
-                addr, last_seen, last_seen_autocrypt, prefer_encrypted,
-                public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
-                gossip_key_fingerprint, verified_key, verified_key_fingerprint
-                */
-                let mut res = Self::new(context, row.get(0)?);
+    async fn from_stmt<'a, P: sqlx::IntoArguments<'a, sqlx::sqlite::Sqlite> + 'a>(
+        context: &Context,
+        query: &'a str,
+        params: P,
+    ) -> Result<Peerstate> {
+        /* all the above queries start with this: SELECT
+        addr, last_seen, last_seen_autocrypt, prefer_encrypted,
+        public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
+        gossip_key_fingerprint, verified_key, verified_key_fingerprint
+        */
+        let peerstate = context.sql.query_row(query, params).await?;
 
-                res.last_seen = row.get(1)?;
-                res.last_seen_autocrypt = row.get(2)?;
-                res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
-                res.gossip_timestamp = row.get(5)?;
-
-                res.public_key_fingerprint = row.get(7)?;
-                if res
-                    .public_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.public_key_fingerprint = None;
-                }
-                res.gossip_key_fingerprint = row.get(8)?;
-                if res
-                    .gossip_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.gossip_key_fingerprint = None;
-                }
-                res.verified_key_fingerprint = row.get(10)?;
-                if res
-                    .verified_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.verified_key_fingerprint = None;
-                }
-                res.public_key = row
-                    .get(4)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
-                res.gossip_key = row
-                    .get(6)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
-                res.verified_key = row
-                    .get(9)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
-
-                Ok(res)
-            })
-            .ok()
+        Ok(peerstate)
     }
 
     pub fn recalc_fingerprint(&mut self) {
@@ -294,8 +260,8 @@ impl<'a> Peerstate<'a> {
                 self.to_save = Some(ToSave::All)
             }
 
-            if self.public_key.as_ref() != Some(&Key::from(header.public_key.clone())) {
-                self.public_key = Some(Key::from(header.public_key.clone()));
+            if self.public_key.as_ref() != Some(&header.public_key) {
+                self.public_key = Some(header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All);
             }
@@ -310,9 +276,8 @@ impl<'a> Peerstate<'a> {
         if message_time > self.gossip_timestamp {
             self.gossip_timestamp = message_time;
             self.to_save = Some(ToSave::Timestamps);
-            let hdr_key = Key::from(gossip_header.public_key.clone());
-            if self.gossip_key.as_ref() != Some(&hdr_key) {
-                self.gossip_key = Some(hdr_key);
+            if self.gossip_key.as_ref() != Some(&gossip_header.public_key) {
+                self.gossip_key = Some(gossip_header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All)
             }
@@ -340,11 +305,9 @@ impl<'a> Peerstate<'a> {
 
     pub fn render_gossip_header(&self, min_verified: PeerstateVerifiedStatus) -> Option<String> {
         if let Some(key) = self.peek_key(min_verified) {
-            // TODO: avoid cloning
-            let public_key = SignedPublicKey::try_from(key.clone()).ok()?;
             let header = Aheader::new(
                 self.addr.clone(),
-                public_key,
+                key.clone(), // TODO: avoid cloning
                 // Autocrypt 1.1.0 specification says that
                 // `prefer-encrypt` attribute SHOULD NOT be included,
                 // but we include it anyway to propagate encryption
@@ -361,7 +324,16 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&Key> {
+    pub fn take_key(mut self, min_verified: PeerstateVerifiedStatus) -> Option<SignedPublicKey> {
+        match min_verified {
+            PeerstateVerifiedStatus::BidirectVerified => self.verified_key.take(),
+            PeerstateVerifiedStatus::Unverified => {
+                self.public_key.take().or_else(|| self.gossip_key.take())
+            }
+        }
+    }
+
+    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&SignedPublicKey> {
         match min_verified {
             PeerstateVerifiedStatus::BidirectVerified => self.verified_key.as_ref(),
             PeerstateVerifiedStatus::Unverified => self
@@ -374,7 +346,7 @@ impl<'a> Peerstate<'a> {
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
         verified: PeerstateVerifiedStatus,
     ) -> bool {
         if verified == PeerstateVerifiedStatus::BidirectVerified {
@@ -409,66 +381,60 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn save_to_db(&self, sql: &Sql, create: bool) -> crate::sql::Result<()> {
+    pub async fn save_to_db(&self, sql: &Sql, create: bool) -> crate::sql::Result<()> {
         if create {
-            sql::execute(
-                self.context,
-                sql,
+            sql.execute(
                 "INSERT INTO acpeerstates (addr) VALUES(?);",
-                params![self.addr],
-            )?;
+                paramsx![&self.addr],
+            )
+            .await?;
         }
 
         if self.to_save == Some(ToSave::All) || create {
-            sql::execute(
-                self.context,
-                sql,
-                "UPDATE acpeerstates \
-                 SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
-                 public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
-                 verified_key=?, verified_key_fingerprint=? \
-                 WHERE addr=?;",
-                params![
+            sql.execute(
+                r#"
+UPDATE acpeerstates 
+  SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?,
+      public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?,
+      verified_key=?, verified_key_fingerprint=?
+  WHERE addr=?;
+"#,
+                paramsx![
                     self.last_seen,
                     self.last_seen_autocrypt,
                     self.prefer_encrypt as i64,
                     self.public_key.as_ref().map(|k| k.to_bytes()),
                     self.gossip_timestamp,
                     self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    &self.public_key_fingerprint,
-                    &self.gossip_key_fingerprint,
+                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    &self.verified_key_fingerprint,
-                    &self.addr,
+                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    &self.addr
                 ],
-            )?;
+            ).await?;
         } else if self.to_save == Some(ToSave::Timestamps) {
-            sql::execute(
-                self.context,
-                sql,
-                "UPDATE acpeerstates SET last_seen=?, last_seen_autocrypt=?, gossip_timestamp=? \
-                 WHERE addr=?;",
-                params![
+            sql.execute(
+                "UPDATE acpeerstates SET last_seen=?, last_seen_autocrypt=?, gossip_timestamp=? WHERE addr=?;",
+                paramsx![
                     self.last_seen,
                     self.last_seen_autocrypt,
                     self.gossip_timestamp,
                     &self.addr
                 ],
-            )?;
+            )
+            .await?;
         }
 
         Ok(())
     }
 
-    pub fn has_verified_key(&self, fingerprints: &HashSet<String>) -> bool {
-        if self.verified_key.is_some() && self.verified_key_fingerprint.is_some() {
-            let vkc = self.verified_key_fingerprint.as_ref().unwrap();
-            if fingerprints.contains(vkc) {
-                return true;
-            }
+    pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
+        if let Some(vkc) = &self.verified_key_fingerprint {
+            fingerprints.contains(vkc) && self.verified_key.is_some()
+        } else {
+            false
         }
-
-        false
     }
 }
 
@@ -477,17 +443,15 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
     use pretty_assertions::assert_eq;
-    use tempfile::TempDir;
 
-    #[test]
-    fn test_peerstate_save_to_db() {
-        let ctx = crate::test_utils::dummy_context();
+    #[async_std::test]
+    async fn test_peerstate_save_to_db() {
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -504,30 +468,30 @@ mod tests {
         };
 
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "failed to save to db"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, addr)
+            .await
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
         assert_eq!(peerstate, peerstate_new);
-        let peerstate_new2 =
-            Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint())
-                .expect("failed to load peerstate from db");
+        let peerstate_new2 = Peerstate::from_fingerprint(&ctx.ctx, &pub_key.fingerprint())
+            .await
+            .expect("failed to load peerstate from db");
         assert_eq!(peerstate, peerstate_new2);
     }
 
-    #[test]
-    fn test_peerstate_double_create() {
-        let ctx = crate::test_utils::dummy_context();
+    #[async_std::test]
+    async fn test_peerstate_double_create() {
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -544,24 +508,23 @@ mod tests {
         };
 
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "failed to save"
         );
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "double-call with create failed"
         );
     }
 
-    #[test]
-    fn test_peerstate_with_empty_gossip_key_save_to_db() {
-        let ctx = crate::test_utils::dummy_context();
+    #[async_std::test]
+    async fn test_peerstate_with_empty_gossip_key_save_to_db() {
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -578,22 +541,16 @@ mod tests {
         };
 
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "failed to save"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, addr)
+            .await
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
         assert_eq!(peerstate, peerstate_new);
     }
-
-    // TODO: don't copy this from stress.rs
-    #[allow(dead_code)]
-    struct TestContext {
-        ctx: Context,
-        dir: TempDir,
-    }
 }

src/pgp.rs

@@ -1,7 +1,6 @@
 //! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
 
 use std::collections::{BTreeMap, HashSet};
-use std::convert::TryInto;
 use std::io;
 use std::io::Cursor;
 
@@ -19,8 +18,8 @@ use rand::{thread_rng, CryptoRng, Rng};
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::key::*;
-use crate::keyring::*;
+use crate::key::{DcKey, Fingerprint};
+use crate::keyring::Keyring;
 
 pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
 pub const HEADER_SETUPCODE: &str = "passphrase-begin";
@@ -119,7 +118,7 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, Str
 /// since all variability is hardcoded.
 #[derive(Debug, thiserror::Error)]
 #[error("PgpKeygenError: {message}")]
-pub(crate) struct PgpKeygenError {
+pub struct PgpKeygenError {
     message: String,
     #[source]
     cause: anyhow::Error,
@@ -238,124 +237,142 @@ fn select_pk_for_encryption(key: &SignedPublicKey) -> Option<SignedPublicKeyOrSu
 
 /// Encrypts `plain` text using `public_keys_for_encryption`
 /// and signs it using `private_key_for_signing`.
-pub fn pk_encrypt(
+pub async fn pk_encrypt(
     plain: &[u8],
-    public_keys_for_encryption: &Keyring,
-    private_key_for_signing: Option<&Key>,
+    public_keys_for_encryption: Keyring<SignedPublicKey>,
+    private_key_for_signing: Option<SignedSecretKey>,
 ) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
-    let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            key.as_ref()
-                .try_into()
-                .ok()
-                .and_then(select_pk_for_encryption)
-        })
-        .collect();
-    let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    let mut rng = thread_rng();
+    async_std::task::spawn_blocking(move || {
+        let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
+            .keys()
+            .iter()
+            .filter_map(|key| select_pk_for_encryption(key))
+            .collect();
+        let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    // TODO: measure time
-    let encrypted_msg = if let Some(private_key) = private_key_for_signing {
-        let skey: &SignedSecretKey = private_key
-            .try_into()
-            .map_err(|_| format_err!("Invalid private key"))?;
+        let mut rng = thread_rng();
 
-        lit_msg
-            .sign(skey, || "".into(), Default::default())
-            .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
-            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
-    } else {
-        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
-    };
+        // TODO: measure time
+        let encrypted_msg = if let Some(ref skey) = private_key_for_signing {
+            lit_msg
+                .sign(skey, || "".into(), Default::default())
+                .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
+                .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
+        } else {
+            lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
+        };
 
-    let msg = encrypted_msg?;
-    let encoded_msg = msg.to_armored_string(None)?;
+        let msg = encrypted_msg?;
+        let encoded_msg = msg.to_armored_string(None)?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
+    .await
 }
 
+/// Decrypts the message with keys from the private key keyring.
+///
+/// Receiver private keys are provided in
+/// `private_keys_for_decryption`.
+///
+/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// of all keys from the `public_keys_for_validation` keyring that
+/// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
-pub fn pk_decrypt(
-    ctext: &[u8],
-    private_keys_for_decryption: &Keyring,
-    public_keys_for_validation: &Keyring,
-    ret_signature_fingerprints: Option<&mut HashSet<String>>,
+pub async fn pk_decrypt(
+    ctext: Vec<u8>,
+    private_keys_for_decryption: Keyring<SignedSecretKey>,
+    public_keys_for_validation: Keyring<SignedPublicKey>,
+    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
 ) -> Result<Vec<u8>> {
-    let (msg, _) = Message::from_armor_single(Cursor::new(ctext))?;
-    let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            let k: &Key = &key;
-            k.try_into().ok()
-        })
-        .collect();
+    let msgs = async_std::task::spawn_blocking(move || {
+        let cursor = Cursor::new(ctext);
+        let (msg, _) = Message::from_armor_single(cursor)?;
 
-    let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-    ensure!(!msgs.is_empty(), "No valid messages found");
+        let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption.keys().iter().collect();
 
-    let dec_msg = &msgs[0];
+        let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
+        decryptor.collect::<pgp::errors::Result<Vec<_>>>()
+    })
+    .await?;
 
-    if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-        if !public_keys_for_validation.keys().is_empty() {
-            let pkeys: Vec<&SignedPublicKey> = public_keys_for_validation
-                .keys()
-                .iter()
-                .filter_map(|key| {
-                    let k: &Key = &key;
-                    k.try_into().ok()
+    if let Some(msg) = msgs.into_iter().next() {
+        // get_content() will decompress the message if needed,
+        // but this avoids decompressing it again to check signatures
+        let msg = msg.decompress()?;
+
+        let content = match msg.get_content()? {
+            Some(content) => content,
+            None => bail!("The decrypted message is empty"),
+        };
+
+        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
+            if !public_keys_for_validation.is_empty() {
+                let fingerprints = async_std::task::spawn_blocking(move || {
+                    let pkeys = public_keys_for_validation.keys();
+
+                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                        for pkey in pkeys {
+                            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                                let fp = DcKey::fingerprint(pkey);
+                                fingerprints.push(fp);
+                            }
+                        }
+                    }
+                    fingerprints
                 })
-                .collect();
+                .await;
 
-            for pkey in &pkeys {
-                if dec_msg.verify(&pkey.primary_key).is_ok() {
-                    let fp = hex::encode_upper(pkey.fingerprint());
-                    ret_signature_fingerprints.insert(fp);
-                }
+                ret_signature_fingerprints.extend(fingerprints);
             }
         }
-    }
-
-    match dec_msg.get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
+        Ok(content)
+    } else {
+        bail!("No valid messages found");
     }
 }
 
 /// Symmetric encryption.
-pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
-    let mut rng = thread_rng();
+pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
+    let passphrase = passphrase.to_string();
 
-    let s2k = StringToKey::new_default(&mut rng);
-    let msg =
-        lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase.into())?;
+    async_std::task::spawn_blocking(move || {
+        let mut rng = thread_rng();
+        let s2k = StringToKey::new_default(&mut rng);
+        let msg =
+            lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase)?;
 
-    let encoded_msg = msg.to_armored_string(None)?;
+        let encoded_msg = msg.to_armored_string(None)?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
+    .await
 }
 
 /// Symmetric decryption.
-pub fn symm_decrypt<T: std::io::Read + std::io::Seek>(
+pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     ctext: T,
 ) -> Result<Vec<u8>> {
     let (enc_msg, _) = Message::from_armor_single(ctext)?;
-    let decryptor = enc_msg.decrypt_with_password(|| passphrase.into())?;
 
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-    ensure!(!msgs.is_empty(), "No valid messages found");
+    let passphrase = passphrase.to_string();
+    async_std::task::spawn_blocking(move || {
+        let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
-    match msgs[0].get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
-    }
+        let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
+        ensure!(!msgs.is_empty(), "No valid messages found");
+
+        match msgs[0].get_content()? {
+            Some(content) => Ok(content),
+            None => bail!("Decrypted message is empty"),
+        }
+    })
+    .await
 }
 
 #[cfg(test)]
@@ -408,10 +425,10 @@ mod tests {
 
     /// [Key] objects to use in tests.
     struct TestKeys {
-        alice_secret: Key,
-        alice_public: Key,
-        bob_secret: Key,
-        bob_public: Key,
+        alice_secret: SignedSecretKey,
+        alice_public: SignedPublicKey,
+        bob_secret: SignedSecretKey,
+        bob_public: SignedPublicKey,
     }
 
     impl TestKeys {
@@ -419,10 +436,10 @@ mod tests {
             let alice = alice_keypair();
             let bob = bob_keypair();
             TestKeys {
-                alice_secret: Key::from(alice.secret.clone()),
-                alice_public: Key::from(alice.public.clone()),
-                bob_secret: Key::from(bob.secret.clone()),
-                bob_public: Key::from(bob.public.clone()),
+                alice_secret: alice.secret.clone(),
+                alice_public: alice.public.clone(),
+                bob_secret: bob.secret.clone(),
+                bob_public: bob.public.clone(),
             }
         }
     }
@@ -436,18 +453,18 @@ mod tests {
 
         /// A cyphertext encrypted to Alice & Bob, signed by Alice.
         static ref CTEXT_SIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, Some(&KEYS.alice_secret)).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
         };
 
         /// A cyphertext encrypted to Alice & Bob, not signed.
         static ref CTEXT_UNSIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, None).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
         };
     }
 
@@ -463,110 +480,115 @@ mod tests {
         assert!(CTEXT_UNSIGNED.starts_with("-----BEGIN PGP MESSAGE-----"));
     }
 
-    #[test]
-    fn test_decrypt_singed() {
+    #[async_std::test]
+    async fn test_decrypt_singed() {
         // Check decrypting as Alice
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring: Keyring<SignedSecretKey> = Keyring::new();
+        decrypt_keyring.add(KEYS.alice_secret.clone());
+        let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
 
         // Check decrypting as Bob
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
     }
 
-    #[test]
-    fn test_decrypt_no_sig_check() {
-        let mut keyring = Keyring::default();
-        keyring.add_ref(&KEYS.alice_secret);
-        let empty_keyring = Keyring::default();
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_no_sig_check() {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_secret.clone());
+        let empty_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &keyring,
-            &empty_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            keyring,
+            empty_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_key() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_key() {
         // The validation does not have the public key of the signer.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.bob_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.bob_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_unsigned() {
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let sig_check_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_unsigned() {
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let sig_check_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_UNSIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_UNSIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_sigret() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_sigret() {
         // Check decrypting signed cyphertext without providing the HashSet for signatures.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             None,
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
     }

src/provider/data.rs

@@ -18,6 +18,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // aol.md: aol.com
@@ -28,6 +31,24 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/aol",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // arcor.de.md: arcor.de
+    static ref P_ARCOR_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/arcor-de",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.arcor.de", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.arcor.de", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // autistici.org.md: autistici.org
@@ -40,6 +61,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mail.autistici.org", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -52,16 +76,82 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imaps.bluewin.ch", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // chello.at.md: chello.at
+    static ref P_CHELLO_AT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/chello-at",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "mail.mymagenta.at", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.mymagenta.at", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_COMCAST: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/comcast",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // dismail.de.md: dismail.de
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISMAIL_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dismail-de",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // disroot.md: disroot.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISROOT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/disroot",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
+
+    // dubby.org.md: dubby.org
+    static ref P_DUBBY_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dubby-org",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // example.com.md: example.com, example.org
     static ref P_EXAMPLE_COM: Provider = Provider {
@@ -73,6 +163,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.example.com", port: 1337, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // fastmail.md: fastmail.com
@@ -83,6 +176,27 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/fastmail",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // five.chat.md: five.chat
+    static ref P_FIVE_CHAT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/five-chat",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // freenet.de.md: freenet.de
@@ -95,6 +209,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mx.freenet.de", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -107,6 +224,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.gmail.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -120,10 +240,36 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "mail.gmx.net", port: 465, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // hey.com.md: hey.com
+    static ref P_HEY_COM: Provider = Provider {
+        status: Status::BROKEN,
+        before_login_hint: "hey.com does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to hey.com.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/hey-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // i.ua.md: i.ua
-    // - skipping provider with status OK and no special things to do
+    static ref P_I_UA: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/i-ua",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // icloud.md: icloud.com, me.com, mac.com
     static ref P_ICLOUD: Provider = Provider {
@@ -135,19 +281,62 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.me.com", port: 993, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // kolst.com.md: kolst.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KOLST_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kolst-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // kontent.com.md: kontent.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KONTENT_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kontent-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAIL_RU: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mail-ru",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAILBOX_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mailbox-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // nauta.cu.md: nauta.cu
     static ref P_NAUTA_CU: Provider = Provider {
@@ -159,6 +348,17 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.nauta.cu", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.nauta.cu", port: 25, username_pattern: EMAIL },
         ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::DeleteServerAfter, value: "1" },
+            ConfigDefault { key: Config::BccSelf, value: "0" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+            ConfigDefault { key: Config::E2eeEnabled, value: "0" },
+            ConfigDefault { key: Config::MediaQuality, value: "1" },
+        ]),
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -171,9 +371,12 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap-mail.outlook.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // posteo.md: posteo.de
+    // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
     static ref P_POSTEO: Provider = Provider {
         status: Status::OK,
         before_login_hint: "",
@@ -183,6 +386,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "posteo.de", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -193,13 +399,49 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/protonmail",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // riseup.net.md: riseup.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_RISEUP_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/riseup-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // rogers.com.md: rogers.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_ROGERS_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/rogers-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // systemli.org.md: systemli.org
+    static ref P_SYSTEMLI_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/systemli-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
@@ -209,6 +451,9 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/t-online",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // testrun.md: testrun.org
@@ -222,6 +467,14 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // tiscali.it.md: tiscali.it
@@ -234,13 +487,36 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.tiscali.it", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // ukr.net.md: ukr.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_UKR_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/ukr-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // vfemail.md: vfemail.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_VFEMAIL: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/vfemail",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
     static ref P_WEB_DE: Provider = Provider {
@@ -253,22 +529,38 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.web.de", port: 143, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
     static ref P_YAHOO: Provider = Provider {
         status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with your Yahoo email address you have to allow \"less secure apps\" in the Yahoo webinterface.",
+        before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yahoo",
         server: vec![
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // yandex.ru.md: yandex.ru, yandex.com
-    // - skipping provider with status OK and no special things to do
+    // yandex.ru.md: yandex.com, yandex.by, yandex.kz, yandex.ru, yandex.ua, ya.ru, narod.ru
+    static ref P_YANDEX_RU: Provider = Provider {
+        status: Status::PREPARATION,
+        before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/yandex-ru",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
+    };
 
     // ziggo.nl.md: ziggo.nl
     static ref P_ZIGGO_NL: Provider = Provider {
@@ -280,16 +572,27 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.ziggo.nl", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
+        ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("chello.at", &*P_CHELLO_AT),
+        ("xfinity.com", &*P_COMCAST),
+        ("comcast.net", &*P_COMCAST),
+        ("dismail.de", &*P_DISMAIL_DE),
+        ("disroot.org", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -302,9 +605,19 @@ lazy_static::lazy_static! {
         ("gmx.info", &*P_GMX_NET),
         ("gmx.biz", &*P_GMX_NET),
         ("gmx.com", &*P_GMX_NET),
+        ("hey.com", &*P_HEY_COM),
+        ("i.ua", &*P_I_UA),
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
+        ("kolst.com", &*P_KOLST_COM),
+        ("kontent.com", &*P_KONTENT_COM),
+        ("mail.ru", &*P_MAIL_RU),
+        ("inbox.ru", &*P_MAIL_RU),
+        ("bk.ru", &*P_MAIL_RU),
+        ("list.ru", &*P_MAIL_RU),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
@@ -312,12 +625,65 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
+        ("posteo.af", &*P_POSTEO),
+        ("posteo.at", &*P_POSTEO),
+        ("posteo.be", &*P_POSTEO),
+        ("posteo.ch", &*P_POSTEO),
+        ("posteo.cl", &*P_POSTEO),
+        ("posteo.co", &*P_POSTEO),
+        ("posteo.co.uk", &*P_POSTEO),
+        ("posteo.com.br", &*P_POSTEO),
+        ("posteo.cr", &*P_POSTEO),
+        ("posteo.cz", &*P_POSTEO),
+        ("posteo.dk", &*P_POSTEO),
+        ("posteo.ee", &*P_POSTEO),
+        ("posteo.es", &*P_POSTEO),
+        ("posteo.eu", &*P_POSTEO),
+        ("posteo.fi", &*P_POSTEO),
+        ("posteo.gl", &*P_POSTEO),
+        ("posteo.gr", &*P_POSTEO),
+        ("posteo.hn", &*P_POSTEO),
+        ("posteo.hr", &*P_POSTEO),
+        ("posteo.hu", &*P_POSTEO),
+        ("posteo.ie", &*P_POSTEO),
+        ("posteo.in", &*P_POSTEO),
+        ("posteo.is", &*P_POSTEO),
+        ("posteo.jp", &*P_POSTEO),
+        ("posteo.la", &*P_POSTEO),
+        ("posteo.li", &*P_POSTEO),
+        ("posteo.lt", &*P_POSTEO),
+        ("posteo.lu", &*P_POSTEO),
+        ("posteo.me", &*P_POSTEO),
+        ("posteo.mx", &*P_POSTEO),
+        ("posteo.my", &*P_POSTEO),
+        ("posteo.net", &*P_POSTEO),
+        ("posteo.nl", &*P_POSTEO),
+        ("posteo.no", &*P_POSTEO),
+        ("posteo.nz", &*P_POSTEO),
+        ("posteo.org", &*P_POSTEO),
+        ("posteo.pe", &*P_POSTEO),
+        ("posteo.pl", &*P_POSTEO),
+        ("posteo.pm", &*P_POSTEO),
+        ("posteo.pt", &*P_POSTEO),
+        ("posteo.ro", &*P_POSTEO),
+        ("posteo.ru", &*P_POSTEO),
+        ("posteo.se", &*P_POSTEO),
+        ("posteo.sg", &*P_POSTEO),
+        ("posteo.si", &*P_POSTEO),
+        ("posteo.tn", &*P_POSTEO),
+        ("posteo.uk", &*P_POSTEO),
+        ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("rogers.com", &*P_ROGERS_COM),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
+        ("ukr.net", &*P_UKR_NET),
+        ("vfemail.net", &*P_VFEMAIL),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),
         ("flirt.ms", &*P_WEB_DE),
@@ -360,6 +726,13 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
+        ("yandex.com", &*P_YANDEX_RU),
+        ("yandex.by", &*P_YANDEX_RU),
+        ("yandex.kz", &*P_YANDEX_RU),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("yandex.ua", &*P_YANDEX_RU),
+        ("ya.ru", &*P_YANDEX_RU),
+        ("narod.ru", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/provider/mod.rs

@@ -2,6 +2,7 @@
 
 mod data;
 
+use crate::config::Config;
 use crate::dc_tools::EmailAddress;
 use crate::provider::data::PROVIDER_DATA;
 
@@ -34,6 +35,13 @@ pub enum UsernamePattern {
     EMAILLOCALPART = 2,
 }
 
+#[derive(Debug, PartialEq)]
+#[repr(u8)]
+pub enum Oauth2Authorizer {
+    Yandex = 1,
+    Gmail = 2,
+}
+
 #[derive(Debug)]
 pub struct Server {
     pub protocol: Protocol,
@@ -57,6 +65,12 @@ impl Server {
     }
 }
 
+#[derive(Debug)]
+pub struct ConfigDefault {
+    pub key: Config,
+    pub value: &'static str,
+}
+
 #[derive(Debug)]
 pub struct Provider {
     pub status: Status,
@@ -64,6 +78,9 @@ pub struct Provider {
     pub after_login_hint: &'static str,
     pub overview_page: &'static str,
     pub server: Vec<Server>,
+    pub config_defaults: Option<Vec<ConfigDefault>>,
+    pub strict_tls: bool,
+    pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
 impl Provider {

src/provider/update.py

@@ -9,6 +9,9 @@ out_all = ""
 out_domains = ""
 domains_dict = {}
 
+def camel(name):
+    words = name.split("_")
+    return "".join(w.capitalize() for i, w in enumerate(words))
 
 def cleanstr(s):
     s = s.strip()
@@ -31,6 +34,18 @@ def file2url(f):
     return "https://providers.delta.chat/" + f
 
 
+def process_config_defaults(data):
+    if not "config_defaults" in data:
+        return "None"
+    defaults = "Some(vec![\n"
+    config_defaults = data.get("config_defaults", "")
+    for key in config_defaults:
+        value = str(config_defaults[key])
+        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
+    defaults += "        ])"
+    return defaults
+
+
 def process_data(data, file):
     status = data.get("status", "")
     if status != "OK" and status != "PREPARATION" and status != "BROKEN":
@@ -83,6 +98,14 @@ def process_data(data, file):
             server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
+    config_defaults = process_config_defaults(data)
+
+    strict_tls = data.get("strict_tls", False)
+    strict_tls = "true" if strict_tls else "false"
+
+    oauth2 = data.get("oauth2", "")
+    oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -93,6 +116,9 @@ def process_data(data, file):
         provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
+        provider += "        config_defaults: " + config_defaults + ",\n"
+        provider += "        strict_tls: " + strict_tls + ",\n"
+        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -103,11 +129,11 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "":
-        out_all += "    // - skipping provider with status OK and no special things to do\n\n"
-    else:
-        out_all += provider
-        out_domains += domains
+
+    # also add provider with no special things to do -
+    # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
+    out_all += provider
+    out_domains += domains
 
 
 def process_file(file):

src/qr.rs

@@ -2,7 +2,6 @@
 
 use lazy_static::lazy_static;
 use percent_encoding::percent_decode_str;
-use reqwest::Url;
 use serde::Deserialize;
 
 use crate::chat;
@@ -11,8 +10,7 @@ use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::key::dc_format_fingerprint;
-use crate::key::dc_normalize_fingerprint;
+use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
@@ -37,26 +35,30 @@ impl Into<Lot> for Error {
     }
 }
 
+fn starts_with_ignore_case(string: &str, pattern: &str) -> bool {
+    string.to_lowercase().starts_with(&pattern.to_lowercase())
+}
+
 /// Check a scanned QR code.
 /// The function should be called after a QR code is scanned.
 /// The function takes the raw text scanned and checks what can be done with it.
-pub fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
+pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
     let qr = qr.as_ref();
 
     info!(context, "Scanned QR code: {}", qr);
 
-    if qr.starts_with(OPENPGP4FPR_SCHEME) {
-        decode_openpgp(context, qr)
-    } else if qr.starts_with(DCACCOUNT_SCHEME) {
+    if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) {
+        decode_openpgp(context, qr).await
+    } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
-        decode_mailto(context, qr)
+        decode_mailto(context, qr).await
     } else if qr.starts_with(SMTP_SCHEME) {
-        decode_smtp(context, qr)
+        decode_smtp(context, qr).await
     } else if qr.starts_with(MATMSG_SCHEME) {
-        decode_matmsg(context, qr)
+        decode_matmsg(context, qr).await
     } else if qr.starts_with(VCARD_SCHEME) {
-        decode_vcard(context, qr)
+        decode_vcard(context, qr).await
     } else if qr.starts_with(HTTP_SCHEME) || qr.starts_with(HTTPS_SCHEME) {
         Lot::from_url(qr)
     } else {
@@ -66,7 +68,7 @@ pub fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
 /// scheme: `OPENPGP4FPR:FINGERPRINT#a=ADDR&n=NAME&i=INVITENUMBER&s=AUTH`
 ///     or: `OPENPGP4FPR:FINGERPRINT#a=ADDR&g=GROUPNAME&x=GROUPID&i=INVITENUMBER&s=AUTH`
-fn decode_openpgp(context: &Context, qr: &str) -> Lot {
+async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
     let (fingerprint, fragment) = match payload.find('#').map(|offset| {
@@ -77,6 +79,14 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         Some(pair) => pair,
         None => (payload, ""),
     };
+    let fingerprint: Fingerprint = match fingerprint.parse() {
+        Ok(fp) => fp,
+        Err(err) => {
+            return Error::new(err)
+                .context("Failed to parse fingerprint in QR code")
+                .into()
+        }
+    };
 
     // replace & with \n to match expected param format
     let fragment = fragment.replace('&', "\n");
@@ -125,25 +135,13 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         None
     };
 
-    let fingerprint = dc_normalize_fingerprint(fingerprint);
-
-    // ensure valid fingerprint
-    if fingerprint.len() != 40 {
-        return format_err!("Bad fingerprint length in QR code").into();
-    }
-
-    println!(
-        "{:?} {:?} {:?} {:?} {:?} {:?} {:?}",
-        addr, name, invitenumber, auth, grpid, grpname, fingerprint
-    );
-
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
-    let peerstate = Peerstate::from_fingerprint(context, &context.sql, &fingerprint);
+    let peerstate = Peerstate::from_fingerprint(context, &fingerprint).await;
 
     if invitenumber.is_none() || auth.is_none() {
-        if let Some(peerstate) = peerstate {
+        if let Ok(peerstate) = peerstate {
             lot.state = LotState::QrFprOk;
 
             lot.id = Contact::add_or_lookup(
@@ -152,16 +150,18 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
                 peerstate.addr.clone(),
                 Origin::UnhandledQrScan,
             )
+            .await
             .map(|(id, _)| id)
             .unwrap_or_default();
 
             let (id, _) = chat::create_or_lookup_by_contact_id(context, lot.id, Blocked::Deaddrop)
+                .await
                 .unwrap_or_default();
 
-            chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr));
+            chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr)).await;
         } else {
             lot.state = LotState::QrFprWithoutAddr;
-            lot.text1 = Some(dc_format_fingerprint(&fingerprint));
+            lot.text1 = Some(fingerprint.to_string());
         }
     } else if let Some(addr) = addr {
         if grpid.is_some() && grpname.is_some() {
@@ -172,6 +172,7 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             lot.state = LotState::QrAskVerifyContact;
         }
         lot.id = Contact::add_or_lookup(context, &name, &addr, Origin::UnhandledQrScan)
+            .await
             .map(|(id, _)| id)
             .unwrap_or_default();
 
@@ -191,7 +192,7 @@ fn decode_account(_context: &Context, qr: &str) -> Lot {
 
     let mut lot = Lot::new();
 
-    if let Ok(url) = Url::parse(payload) {
+    if let Ok(url) = url::Url::parse(payload) {
         if url.scheme() == "https" {
             lot.state = LotState::QrAccount;
             lot.text1 = url.host_str().map(|x| x.to_string());
@@ -216,31 +217,22 @@ struct CreateAccountResponse {
 /// take a qr of the type DC_QR_ACCOUNT, parse it's parameters,
 /// download additional information from the contained url and set the parameters.
 /// on success, a configure::configure() should be able to log in to the account
-pub fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
 
-    let response = reqwest::blocking::Client::new().post(url_str).send();
+    let response: Result<CreateAccountResponse, surf::Error> =
+        surf::post(url_str).recv_json().await;
     if response.is_err() {
         bail!("Cannot create account, request to {} failed", url_str);
     }
-    let response = response.unwrap();
-    if !response.status().is_success() {
-        bail!("Request to {} unsuccessful: {:?}", url_str, response);
-    }
+    let parsed = response.unwrap();
 
-    let parsed: reqwest::Result<CreateAccountResponse> = response.json();
-    if parsed.is_err() {
-        bail!(
-            "Failed to parse JSON response from {}: error: {:?}",
-            url_str,
-            parsed
-        );
-    }
-    println!("response: {:?}", &parsed);
-    let parsed = parsed.unwrap();
-
-    context.set_config(Config::Addr, Some(&parsed.email))?;
-    context.set_config(Config::MailPw, Some(&parsed.password))?;
+    context
+        .set_config(Config::Addr, Some(&parsed.email))
+        .await?;
+    context
+        .set_config(Config::MailPw, Some(&parsed.password))
+        .await?;
 
     Ok(())
 }
@@ -248,7 +240,7 @@ pub fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
 /// Extract address for the mailto scheme.
 ///
 /// Scheme: `mailto:addr...?subject=...&body=..`
-fn decode_mailto(context: &Context, qr: &str) -> Lot {
+async fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let payload = &qr[MAILTO_SCHEME.len()..];
 
     let addr = if let Some(query_index) = payload.find('?') {
@@ -263,13 +255,13 @@ fn decode_mailto(context: &Context, qr: &str) -> Lot {
     };
 
     let name = "".to_string();
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 /// Extract address for the smtp scheme.
 ///
 /// Scheme: `SMTP:addr...:subject...:body...`
-fn decode_smtp(context: &Context, qr: &str) -> Lot {
+async fn decode_smtp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[SMTP_SCHEME.len()..];
 
     let addr = if let Some(query_index) = payload.find(':') {
@@ -283,7 +275,7 @@ fn decode_smtp(context: &Context, qr: &str) -> Lot {
         Err(err) => return err.into(),
     };
     let name = "".to_string();
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 /// Extract address for the matmsg scheme.
@@ -291,7 +283,7 @@ fn decode_smtp(context: &Context, qr: &str) -> Lot {
 /// Scheme: `MATMSG:TO:addr...;SUB:subject...;BODY:body...;`
 ///
 /// There may or may not be linebreaks after the fields.
-fn decode_matmsg(context: &Context, qr: &str) -> Lot {
+async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     // Does not work when the text `TO:` is used in subject/body _and_ TO: is not the first field.
     // we ignore this case.
     let addr = if let Some(to_index) = qr.find("TO:") {
@@ -311,7 +303,7 @@ fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     };
 
     let name = "".to_string();
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 lazy_static! {
@@ -324,7 +316,7 @@ lazy_static! {
 /// Extract address for the matmsg scheme.
 ///
 /// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
-fn decode_vcard(context: &Context, qr: &str) -> Lot {
+async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
         .captures(qr)
         .map(|caps| {
@@ -344,7 +336,7 @@ fn decode_vcard(context: &Context, qr: &str) -> Lot {
         return format_err!("Bad e-mail address").into();
     };
 
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 impl Lot {
@@ -364,10 +356,10 @@ impl Lot {
         l
     }
 
-    pub fn from_address(context: &Context, name: String, addr: String) -> Self {
+    pub async fn from_address(context: &Context, name: String, addr: String) -> Self {
         let mut l = Lot::new();
         l.state = LotState::QrAddr;
-        l.id = match Contact::add_or_lookup(context, name, addr, Origin::UnhandledQrScan) {
+        l.id = match Contact::add_or_lookup(context, name, addr, Origin::UnhandledQrScan).await {
             Ok((id, _)) => id,
             Err(err) => return err.into(),
         };
@@ -391,13 +383,13 @@ fn normalize_address(addr: &str) -> Result<String, Error> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
-    #[test]
-    fn test_decode_http() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_http() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "http://www.hello.com");
+        let res = check_qr(&ctx.ctx, "http://www.hello.com").await;
 
         assert_eq!(res.get_state(), LotState::QrUrl);
         assert_eq!(res.get_id(), 0);
@@ -405,11 +397,11 @@ mod tests {
         assert!(res.get_text2().is_none());
     }
 
-    #[test]
-    fn test_decode_https() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_https() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "https://www.hello.com");
+        let res = check_qr(&ctx.ctx, "https://www.hello.com").await;
 
         assert_eq!(res.get_state(), LotState::QrUrl);
         assert_eq!(res.get_id(), 0);
@@ -417,11 +409,11 @@ mod tests {
         assert!(res.get_text2().is_none());
     }
 
-    #[test]
-    fn test_decode_text() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_text() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "I am so cool");
+        let res = check_qr(&ctx.ctx, "I am so cool").await;
 
         assert_eq!(res.get_state(), LotState::QrText);
         assert_eq!(res.get_id(), 0);
@@ -429,124 +421,148 @@ mod tests {
         assert!(res.get_text2().is_none());
     }
 
-    #[test]
-    fn test_decode_vcard() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_vcard() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "BEGIN:VCARD\nVERSION:3.0\nN:Last;First\nEMAIL;TYPE=INTERNET:stress@test.local\nEND:VCARD"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
         assert_eq!(contact.get_name(), "First Last");
     }
 
-    #[test]
-    fn test_decode_matmsg() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_matmsg() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "MATMSG:TO:\n\nstress@test.local ; \n\nSUB:\n\nSubject here\n\nBODY:\n\nhelloworld\n;;",
-        );
+        )
+        .await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
     }
 
-    #[test]
-    fn test_decode_mailto() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_mailto() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "mailto:stress@test.local?subject=hello&body=world",
-        );
+        )
+        .await;
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
 
-        let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org");
+        let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org").await;
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "no-questionmark@example.org");
 
-        let res = check_qr(&ctx.ctx, "mailto:no-addr");
+        let res = check_qr(&ctx.ctx, "mailto:no-addr").await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
     }
 
-    #[test]
-    fn test_decode_smtp() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_smtp() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld");
+        let res = check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld").await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
     }
 
-    #[test]
-    fn test_decode_openpgp_group() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_openpgp_group() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
         assert_ne!(res.get_id(), 0);
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        // Test it again with lowercased "openpgp4fpr:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await;
+
+        println!("{:?}", res);
+        assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
+        assert_ne!(res.get_id(), 0);
+        assert_eq!(res.get_text1().unwrap(), "test ? test !");
+
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
     }
 
-    #[test]
-    fn test_decode_openpgp_secure_join() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_openpgp_secure_join() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        // Test it again with lowercased "openpgp4fpr:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+        ).await;
+
+        println!("{:?}", res);
+        assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
+        assert_ne!(res.get_id(), 0);
+
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
     }
 
-    #[test]
-    fn test_decode_openpgp_without_addr() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_openpgp_without_addr() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "OPENPGP4FPR:1234567890123456789012345678901234567890",
-        );
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
         assert_eq!(
             res.get_text1().unwrap(),
@@ -554,31 +570,64 @@ mod tests {
         );
         assert_eq!(res.get_id(), 0);
 
-        let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890");
+        // Test it again with lowercased "openpgp4fpr:" uri scheme
+
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:1234567890123456789012345678901234567890",
+        )
+        .await;
+        assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
+        assert_eq!(
+            res.get_text1().unwrap(),
+            "1234 5678 9012 3456 7890\n1234 5678 9012 3456 7890"
+        );
+        assert_eq!(res.get_id(), 0);
+
+        let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890").await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert_eq!(res.get_id(), 0);
     }
 
-    #[test]
-    fn test_decode_account() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_account() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
+        )
+        .await;
+        assert_eq!(res.get_state(), LotState::QrAccount);
+        assert_eq!(res.get_text1().unwrap(), "example.org");
+
+        // Test it again with lowercased "dcaccount:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "dcaccount:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrAccount);
         assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
-    #[test]
-    fn test_decode_account_bad_scheme() {
-        let ctx = dummy_context();
-
+    #[async_std::test]
+    async fn test_decode_account_bad_scheme() {
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
             "DCACCOUNT:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
+        )
+        .await;
+        assert_eq!(res.get_state(), LotState::QrError);
+        assert!(res.get_text1().is_some());
+
+        // Test it again with lowercased "dcaccount:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
     }

src/scheduler.rs

@@ -0,0 +1,550 @@
+#![warn(clippy::indexing_slicing)]
+
+use async_std::prelude::*;
+use async_std::sync::{channel, Receiver, Sender};
+use async_std::task;
+
+use crate::context::Context;
+use crate::imap::Imap;
+use crate::job::{self, Thread};
+use crate::{config::Config, message::MsgId, smtp::Smtp};
+
+pub(crate) struct StopToken;
+
+/// Job and connection scheduler.
+#[derive(Debug)]
+#[allow(clippy::large_enum_variant)]
+pub(crate) enum Scheduler {
+    Stopped,
+    Running {
+        inbox: ImapConnectionState,
+        inbox_handle: Option<task::JoinHandle<()>>,
+        mvbox: ImapConnectionState,
+        mvbox_handle: Option<task::JoinHandle<()>>,
+        sentbox: ImapConnectionState,
+        sentbox_handle: Option<task::JoinHandle<()>>,
+        smtp: SmtpConnectionState,
+        smtp_handle: Option<task::JoinHandle<()>>,
+    },
+}
+
+impl Context {
+    /// Indicate that the network likely has come back.
+    pub async fn maybe_network(&self) {
+        self.scheduler.read().await.maybe_network().await;
+    }
+
+    pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_inbox(info).await;
+    }
+
+    pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_smtp(info).await;
+    }
+}
+
+async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers) {
+    use futures::future::FutureExt;
+
+    info!(ctx, "starting inbox loop");
+    let ImapConnectionHandlers {
+        mut connection,
+        stop_receiver,
+        shutdown_sender,
+    } = inbox_handlers;
+
+    let ctx1 = ctx.clone();
+    let fut = async move {
+        started.send(()).await;
+        let ctx = ctx1;
+
+        // track number of continously executed jobs
+        let mut jobs_loaded = 0;
+        let mut info = InterruptInfo::default();
+        loop {
+            match job::load_next(&ctx, Thread::Imap, &info).await {
+                Some(job) if jobs_loaded <= 20 => {
+                    jobs_loaded += 1;
+                    job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
+                    info = Default::default();
+                }
+                Some(job) => {
+                    // Let the fetch run, but return back to the job afterwards.
+                    info!(ctx, "postponing imap-job {} to run fetch...", job);
+                    jobs_loaded = 0;
+                    fetch(&ctx, &mut connection).await;
+                }
+                None => {
+                    jobs_loaded = 0;
+                    info = if ctx.get_config_bool(Config::InboxWatch).await {
+                        fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
+                    } else {
+                        connection.fake_idle(&ctx, None).await
+                    };
+                }
+            }
+        }
+    };
+
+    stop_receiver
+        .recv()
+        .map(|_| {
+            info!(ctx, "shutting down inbox loop");
+        })
+        .race(fut)
+        .await;
+    shutdown_sender.send(()).await;
+}
+
+async fn fetch(ctx: &Context, connection: &mut Imap) {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
+        Some(watch_folder) => {
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "{}", err);
+                return;
+            }
+
+            // fetch
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
+        }
+        None => {
+            warn!(ctx, "Can not fetch inbox folder, not set");
+            connection.fake_idle(&ctx, None).await;
+        }
+    }
+}
+
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
+    match ctx.get_config(folder).await {
+        Some(watch_folder) => {
+            // connect and fake idle if unable to connect
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "imap connection failed: {}", err);
+                return connection.fake_idle(&ctx, None).await;
+            }
+
+            // fetch
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
+
+            // idle
+            if connection.can_idle() {
+                connection
+                    .idle(&ctx, Some(watch_folder))
+                    .await
+                    .unwrap_or_else(|err| {
+                        connection.trigger_reconnect();
+                        warn!(ctx, "{}", err);
+                        InterruptInfo::new(false, None)
+                    })
+            } else {
+                connection.fake_idle(&ctx, Some(watch_folder)).await
+            }
+        }
+        None => {
+            warn!(ctx, "Can not watch {} folder, not set", folder);
+            connection.fake_idle(&ctx, None).await
+        }
+    }
+}
+
+async fn simple_imap_loop(
+    ctx: Context,
+    started: Sender<()>,
+    inbox_handlers: ImapConnectionHandlers,
+    folder: Config,
+) {
+    use futures::future::FutureExt;
+
+    info!(ctx, "starting simple loop for {}", folder.as_ref());
+    let ImapConnectionHandlers {
+        mut connection,
+        stop_receiver,
+        shutdown_sender,
+    } = inbox_handlers;
+
+    let ctx1 = ctx.clone();
+
+    let fut = async move {
+        started.send(()).await;
+        let ctx = ctx1;
+
+        loop {
+            fetch_idle(&ctx, &mut connection, folder).await;
+        }
+    };
+
+    stop_receiver
+        .recv()
+        .map(|_| {
+            info!(ctx, "shutting down simple loop");
+        })
+        .race(fut)
+        .await;
+    shutdown_sender.send(()).await;
+}
+
+async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
+    use futures::future::FutureExt;
+
+    info!(ctx, "starting smtp loop");
+    let SmtpConnectionHandlers {
+        mut connection,
+        stop_receiver,
+        shutdown_sender,
+        idle_interrupt_receiver,
+    } = smtp_handlers;
+
+    let ctx1 = ctx.clone();
+    let fut = async move {
+        started.send(()).await;
+        let ctx = ctx1;
+
+        let mut interrupt_info = Default::default();
+        loop {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
+                Some(job) => {
+                    info!(ctx, "executing smtp job");
+                    job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
+                    interrupt_info = Default::default();
+                }
+                None => {
+                    // Fake Idle
+                    info!(ctx, "smtp fake idle - started");
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    info!(ctx, "smtp fake idle - interrupted")
+                }
+            }
+        }
+    };
+
+    stop_receiver
+        .recv()
+        .map(|_| {
+            info!(ctx, "shutting down smtp loop");
+        })
+        .race(fut)
+        .await;
+    shutdown_sender.send(()).await;
+}
+
+impl Scheduler {
+    /// Start the scheduler, panics if it is already running.
+    pub async fn start(&mut self, ctx: Context) {
+        let (mvbox, mvbox_handlers) = ImapConnectionState::new();
+        let (sentbox, sentbox_handlers) = ImapConnectionState::new();
+        let (smtp, smtp_handlers) = SmtpConnectionState::new();
+        let (inbox, inbox_handlers) = ImapConnectionState::new();
+
+        let (inbox_start_send, inbox_start_recv) = channel(1);
+        let (mvbox_start_send, mvbox_start_recv) = channel(1);
+        let mut mvbox_handle = None;
+        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let mut sentbox_handle = None;
+        let (smtp_start_send, smtp_start_recv) = channel(1);
+
+        let ctx1 = ctx.clone();
+        let inbox_handle = Some(task::spawn(async move {
+            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
+        }));
+
+        if ctx.get_config_bool(Config::MvboxWatch).await {
+            let ctx1 = ctx.clone();
+            mvbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    mvbox_start_send,
+                    mvbox_handlers,
+                    Config::ConfiguredMvboxFolder,
+                )
+                .await
+            }));
+        } else {
+            mvbox_start_send.send(()).await;
+        }
+
+        if ctx.get_config_bool(Config::SentboxWatch).await {
+            let ctx1 = ctx.clone();
+            sentbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    sentbox_start_send,
+                    sentbox_handlers,
+                    Config::ConfiguredSentboxFolder,
+                )
+                .await
+            }));
+        } else {
+            sentbox_start_send.send(()).await;
+        }
+
+        let ctx1 = ctx.clone();
+        let smtp_handle = Some(task::spawn(async move {
+            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
+        }));
+
+        *self = Scheduler::Running {
+            inbox,
+            mvbox,
+            sentbox,
+            smtp,
+            inbox_handle,
+            mvbox_handle,
+            sentbox_handle,
+            smtp_handle,
+        };
+
+        // wait for all loops to be started
+        if let Err(err) = inbox_start_recv
+            .recv()
+            .try_join(mvbox_start_recv.recv())
+            .try_join(sentbox_start_recv.recv())
+            .try_join(smtp_start_recv.recv())
+            .await
+        {
+            error!(ctx, "failed to start scheduler: {}", err);
+        }
+
+        info!(ctx, "scheduler is running");
+    }
+
+    async fn maybe_network(&self) {
+        if !self.is_running() {
+            return;
+        }
+
+        self.interrupt_inbox(InterruptInfo::new(true, None))
+            .join(self.interrupt_mvbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_smtp(InterruptInfo::new(true, None)))
+            .await;
+    }
+
+    async fn interrupt_inbox(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref inbox, .. } = self {
+            inbox.interrupt(info).await;
+        }
+    }
+
+    async fn interrupt_mvbox(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref mvbox, .. } = self {
+            mvbox.interrupt(info).await;
+        }
+    }
+
+    async fn interrupt_sentbox(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref sentbox, .. } = self {
+            sentbox.interrupt(info).await;
+        }
+    }
+
+    async fn interrupt_smtp(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref smtp, .. } = self {
+            smtp.interrupt(info).await;
+        }
+    }
+
+    /// Halts the scheduler, must be called first, and then `stop`.
+    pub(crate) async fn pre_stop(&self) -> StopToken {
+        match self {
+            Scheduler::Stopped => {
+                panic!("WARN: already stopped");
+            }
+            Scheduler::Running {
+                inbox,
+                mvbox,
+                sentbox,
+                smtp,
+                ..
+            } => {
+                inbox
+                    .stop()
+                    .join(mvbox.stop())
+                    .join(sentbox.stop())
+                    .join(smtp.stop())
+                    .await;
+
+                StopToken
+            }
+        }
+    }
+
+    /// Halt the scheduler, must only be called after pre_stop.
+    pub(crate) async fn stop(&mut self, _t: StopToken) {
+        match self {
+            Scheduler::Stopped => {
+                panic!("WARN: already stopped");
+            }
+            Scheduler::Running {
+                inbox_handle,
+                mvbox_handle,
+                sentbox_handle,
+                smtp_handle,
+                ..
+            } => {
+                if let Some(handle) = inbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = mvbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = sentbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = smtp_handle.take() {
+                    handle.await;
+                }
+
+                *self = Scheduler::Stopped;
+            }
+        }
+    }
+
+    /// Check if the scheduler is running.
+    pub fn is_running(&self) -> bool {
+        match self {
+            Scheduler::Running { .. } => true,
+            _ => false,
+        }
+    }
+}
+
+/// Connection state logic shared between imap and smtp connections.
+#[derive(Debug)]
+struct ConnectionState {
+    /// Channel to notify that shutdown has completed.
+    shutdown_receiver: Receiver<()>,
+    /// Channel to interrupt the whole connection.
+    stop_sender: Sender<()>,
+    /// Channel to interrupt idle.
+    idle_interrupt_sender: Sender<InterruptInfo>,
+}
+
+impl ConnectionState {
+    /// Shutdown this connection completely.
+    async fn stop(&self) {
+        // Trigger shutdown of the run loop.
+        self.stop_sender.send(()).await;
+        // Wait for a notification that the run loop has been shutdown.
+        self.shutdown_receiver.recv().await.ok();
+    }
+
+    async fn interrupt(&self, info: InterruptInfo) {
+        // Use try_send to avoid blocking on interrupts.
+        self.idle_interrupt_sender.try_send(info).ok();
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct SmtpConnectionState {
+    state: ConnectionState,
+}
+
+impl SmtpConnectionState {
+    fn new() -> (Self, SmtpConnectionHandlers) {
+        let (stop_sender, stop_receiver) = channel(1);
+        let (shutdown_sender, shutdown_receiver) = channel(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+
+        let handlers = SmtpConnectionHandlers {
+            connection: Smtp::new(),
+            stop_receiver,
+            shutdown_sender,
+            idle_interrupt_receiver,
+        };
+
+        let state = ConnectionState {
+            idle_interrupt_sender,
+            shutdown_receiver,
+            stop_sender,
+        };
+
+        let conn = SmtpConnectionState { state };
+
+        (conn, handlers)
+    }
+
+    /// Interrupt any form of idle.
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
+    }
+
+    /// Shutdown this connection completely.
+    async fn stop(&self) {
+        self.state.stop().await;
+    }
+}
+
+struct SmtpConnectionHandlers {
+    connection: Smtp,
+    stop_receiver: Receiver<()>,
+    shutdown_sender: Sender<()>,
+    idle_interrupt_receiver: Receiver<InterruptInfo>,
+}
+
+#[derive(Debug)]
+pub(crate) struct ImapConnectionState {
+    state: ConnectionState,
+}
+
+impl ImapConnectionState {
+    /// Construct a new connection.
+    fn new() -> (Self, ImapConnectionHandlers) {
+        let (stop_sender, stop_receiver) = channel(1);
+        let (shutdown_sender, shutdown_receiver) = channel(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+
+        let handlers = ImapConnectionHandlers {
+            connection: Imap::new(idle_interrupt_receiver),
+            stop_receiver,
+            shutdown_sender,
+        };
+
+        let state = ConnectionState {
+            idle_interrupt_sender,
+            shutdown_receiver,
+            stop_sender,
+        };
+
+        let conn = ImapConnectionState { state };
+
+        (conn, handlers)
+    }
+
+    /// Interrupt any form of idle.
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
+    }
+
+    /// Shutdown this connection completely.
+    async fn stop(&self) {
+        self.state.stop().await;
+    }
+}
+
+#[derive(Debug)]
+struct ImapConnectionHandlers {
+    connection: Imap,
+    stop_receiver: Receiver<()>,
+    shutdown_sender: Sender<()>,
+}
+
+#[derive(Default, Debug)]
+pub struct InterruptInfo {
+    pub probe_network: bool,
+    pub msg_id: Option<MsgId>,
+}
+
+impl InterruptInfo {
+    pub fn new(probe_network: bool, msg_id: Option<MsgId>) -> Self {
+        Self {
+            probe_network,
+            msg_id,
+        }
+    }
+}