more type fixes

add more tests for BlobObject::sanitise_name()

.circleci/config.yml

@@ -7,6 +7,10 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
+            
 
 restore-workspace: &restore-workspace
   attach_workspace:
@@ -134,12 +138,6 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests_rust:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_rust.sh
-    
   remote_tests_python:
     machine: true
     steps:
@@ -174,11 +172,6 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
-
       - remote_tests_python:
           filters:
             tags:
@@ -187,8 +180,9 @@ workflows:
       - remote_python_packaging:
           requires: 
              - remote_tests_python
-             - remote_tests_rust 
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 
@@ -197,6 +191,8 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -208,6 +204,8 @@ workflows:
 
       - build_doxygen:
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: Rust CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.43.1
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.43.1
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly, 1.43.1]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      with:
+        command:  check
+        args: --workspace --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --workspace
+          

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-12
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-12
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-12
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

ASYNC-API-TODO.txt

@@ -0,0 +1,54 @@
+
+Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
+
+- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
+
+APIs:
+    dc_context_new                 # opens the database
+    dc_open                             # FFI only
+      -> drop it and move parameters to dc_context_new()
+
+    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
+    dc_imex NEVER goes through the job system
+    dc_imex import_backup needs to ensure dc_stop_jobs()
+
+    dc_start_io                # start smtp/imap and job handling subsystems
+    dc_stop_io                 # stop smtp/imap and job handling subsystems
+    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
+
+    dc_close                            # FFI only
+      -> can be dropped
+    dc_context_unref
+
+    for ios share-extension:
+    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
+        0: message was sent
+        1: message failed to go out, is queued as a job to be retried later
+        2: message permanently failed?
+
+EVENT handling:
+    start a callback thread and call get_next_event() which is BLOCKING
+    it's fine to start this callback thread later, it will see all events.
+    Note that the core infinitely fills the internal queue if you never drain it.
+
+    FFI-get_next_event() returns NULL if the context is unrefed already?
+
+   sidenote: how python's callback thread does it currently:
+         CB-thread runs this while loop:
+         while not QUITFLAG:
+             ev = context.get_next_event( )
+             ...
+         So in order to shutdown properly one has to set QUITFLAG
+         before calling dc_stop_jobs() and dc_context_unref
+
+    event API:
+        get_data1_int
+        get_data2_int
+        get_data3_str
+
+
+- userdata likely only used for the callbacks, likely can be dropped, needs verification
+
+
+- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
+

CHANGELOG.md

@@ -1,5 +1,130 @@
 # Changelog 
 
+## 1.39.0
+
+- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
+
+- fix potential panics, update dependencies #1650 #1655
+
+
+## 1.38.0
+
+- fix sorting, esp. for multi-device
+
+
+## 1.37.0
+
+- improve ndn heuristics #1630
+
+- get oauth2 authorizer from provider-db #1641
+
+- removed linebreaks and spaces from generated qr-code #1631
+
+- more fixes #1633 #1635 #1636 #1637
+
+
+## 1.36.0
+
+- parse ndn (network delivery notification) reports
+  and report failed messages as such #1552 #1622 #1630
+
+- add oauth2 support for gsuite domains #1626
+
+- read image orientation from exif before recoding #1619
+
+- improve logging #1593 #1598
+
+- improve python and bot bindings #1583 #1609
+
+- improve imap logout #1595
+
+- fix sorting #1600 #1604
+
+- fix qr code generation #1631
+
+- update rustcrypto releases #1603
+
+- refactorings #1617
+
+
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
+## 1.34.0
+
+- new api for io, thread and event handling #1356,
+  see the example atop of `deltachat.h` to get an overview
+
+- LOTS of speed improvements due to async processing #1356
+
+- enable WAL mode for sqlite #1492
+
+- process incoming messages in bulk #1527
+
+- improve finding out the sent-folder #1488
+
+- several bug fixes
+
+
+## 1.33.0
+
+- let `dc_set_muted()` also mute one-to-one chats #1470
+
+- fix a bug that led to load and traffic if the server does not use sent-folder
+  #1472
+
+
+## 1.32.0
+
+- fix endless loop when trying to download messages with bad RFC Message-ID,
+  also be more reliable on similar errors #1463 #1466 #1462
+
+- fix bug with comma in contact request #1438
+
+- do not refer to hidden messages on replies #1459
+
+- improve error handling #1468 #1465 #1464
+
+
+## 1.31.0
+
+- always describe the context of the displayed error #1451
+
+- do not emit `DC_EVENT_ERROR` when message sending fails;
+  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
+
+- new config-option `media_quality` #1449
+
+- try over if writing message to database fails #1447
+
+
+## 1.30.0
+
+- expunge deleted messages #1440
+
+- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
+
+
 ## 1.29.0
 
 - new config options `delete_device_after` and `delete_server_after`,

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.29.0"
+version = "1.39.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,60 +12,67 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-reqwest = { version = "0.10.0", features = ["blocking", "json"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = "0.2"
+async-smtp = { version = "0.3" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.2"
-async-native-tls = "0.3.1"
-async-std = { version = "1.4", features = ["unstable"] }
-base64 = "0.11"
+async-imap = "0.3.1"
+async-native-tls = { version = "0.3.3" }
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
+kamadak-exif = "0.5"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.21", features = ["bundled"] }
-r2d2_sqlite = "0.13.0"
-r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
-thread-local-object = "0.1.0"
+strum = "0.18.0"
+strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.17.1"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.0"
+mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
-pretty_env_logger = "0.3.1"
-
-rustyline = { version = "4.1.0", optional = true }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
+async-trait = "0.1.31"
+url = "2.1.1"
+async-std-resolver = "0.19.5"
+sqlx = { git = "https://github.com/launchbadge/sqlx", branch = "master", features = ["runtime-async-std", "sqlite", "macros"] }
+libsqlite3-sys = { version = "0.18", features = ["bundled", "min_sqlite_version_3_7_16"] }
+
+pretty_env_logger = { version = "0.4.0", optional = true }
+log = { version = "0.4.8", optional = true }
+rustyline = { version = "4.1.0", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
+async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+smol = "0.1.11"
+log = "0.4.8"
 
 [workspace]
 members = [
@@ -76,15 +83,17 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
+required-features = ["repl"]
 
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
-required-features = ["rustyline"]
+required-features = ["repl"]
 
 
 [features]
-default = ["nightly"]
-vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
+default = []
+internals = []
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
-

README.md

@@ -9,7 +9,7 @@
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
 
 ```
-curl https://sh.rustup.rs -sSf | sh
+$ curl https://sh.rustup.rs -sSf | sh
 ```
 
 ## Using the CLI client
@@ -17,7 +17,7 @@ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-cargo run --example repl -- ~/deltachat-db
+$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -123,6 +123,7 @@ Language bindings are available for:
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
 - [Go](https://github.com/hugot/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library
@@ -132,4 +133,5 @@ or its language bindings:
 - [iOS](https://github.com/deltachat/deltachat-ios)
 - [Desktop](https://github.com/deltachat/deltachat-desktop)
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - several **Bots**

appveyor.yml

@@ -1,19 +0,0 @@
-environment:
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-12
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - rustc -vV
-  - cargo -vV
-
-build: false
-
-test_script:
-  - cargo test --release --all
-
-cache:
-  - target
-  - C:\Users\appveyor\.cargo\registry

ci_scripts/ci_upload.sh

@@ -37,7 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip
+pip3 install -U pip setuptools
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN

ci_scripts/cleanup_devpi_indices.py

@@ -48,7 +48,7 @@ def run():
         projectnames = get_projectnames(baseurl, username, indexname)
         if indexname == "master" or not indexname:
             continue
-        assert projectnames == ["deltachat"]
+        clear_index = not projectnames
         for projectname in projectnames:
             dates = get_release_dates(baseurl, username, indexname, projectname)
             if not dates:
@@ -60,8 +60,11 @@ def run():
                 date = datetime.datetime(*max(dates))
             if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
                 assert username and indexname
-                url = baseurl + username + "/" + indexname
-                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+                clear_index = True
+                break
+        if clear_index:
+            url = baseurl + username + "/" + indexname
+            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
 
 
 

ci_scripts/docker-coredeps/Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/pypa/manylinux1_x86_64
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/
+rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share

ci_scripts/remote_python_packaging.sh

@@ -46,6 +46,6 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
 mkdir -p workspace 
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux1*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
 rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
 rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/run_all.sh

@@ -21,7 +21,8 @@ export DCC_RS_DEV=$(pwd)
 export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
+rm -f python3.5
+ln -s /opt/python/cp35-cp35m/bin/python3.5
 ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
 ln -s /opt/python/cp38-cp38/bin/python3.8

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.29.0"
+version = "1.39.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,10 +20,12 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
+async-std = "1.6.0"
 anyhow = "1.0.28"
 thiserror = "1.0.14"
 
 [features]
-default = ["vendored", "nightly"]
+default = ["vendored"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
+

deltachat_derive/src/lib.rs

@@ -8,36 +8,33 @@ use quote::quote;
 // data.  If this assumption is violated, compiler error will point to
 // generated code, which is not very user-friendly.
 
-#[proc_macro_derive(ToSql)]
-pub fn to_sql_derive(input: TokenStream) -> TokenStream {
+#[proc_macro_derive(Sqlx)]
+pub fn sqlx_derive(input: TokenStream) -> TokenStream {
     let ast: syn::DeriveInput = syn::parse(input).unwrap();
     let name = &ast.ident;
 
     let gen = quote! {
-        impl rusqlite::types::ToSql for #name {
-            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-                let num = *self as i64;
-                let value = rusqlite::types::Value::Integer(num);
-                let output = rusqlite::types::ToSqlOutput::Owned(value);
-                std::result::Result::Ok(output)
-            }
-        }
-    };
-    gen.into()
-}
-
-#[proc_macro_derive(FromSql)]
-pub fn from_sql_derive(input: TokenStream) -> TokenStream {
-    let ast: syn::DeriveInput = syn::parse(input).unwrap();
-    let name = &ast.ident;
-
-    let gen = quote! {
-        impl rusqlite::types::FromSql for #name {
-            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-                let inner = rusqlite::types::FromSql::column_result(col)?;
-                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
+        impl<'q> sqlx::encode::Encode<'q, sqlx::sqlite::Sqlite> for #name {
+            fn encode_by_ref(&self, buf: &mut Vec<sqlx::sqlite::SqliteArgumentValue<'q>>) -> sqlx::encode::IsNull {
+                num_traits::ToPrimitive::to_i32(self).expect("invalid type").encode(buf)
             }
         }
+
+
+        impl<'de> sqlx::decode::Decode<'de, sqlx::sqlite::Sqlite> for #name {
+            fn decode(value: sqlx::sqlite::SqliteValueRef) -> std::result::Result<Self, Box<dyn std::error::Error + 'static + Send + Sync>> {
+                let raw: i32 = sqlx::decode::Decode::decode(value)?;
+
+                Ok(num_traits::FromPrimitive::from_i32(raw).unwrap_or_default())
+            }
+        }
+
+        impl sqlx::types::Type<sqlx::sqlite::Sqlite> for #name {
+            fn type_info() -> sqlx::sqlite::SqliteTypeInfo {
+                <i32 as sqlx::types::Type<_>>::type_info()
+            }
+        }
+
     };
     gen.into()
 }

examples/repl/cmdline.rs

@@ -1,7 +1,7 @@
-use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::{bail, ensure};
+use async_std::path::Path;
 use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
@@ -11,7 +11,6 @@ use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
 use deltachat::error::Error;
 use deltachat::imex::*;
-use deltachat::job::*;
 use deltachat::location;
 use deltachat::lot::LotState;
 use deltachat::message::{self, Message, MessageState, MsgId};
@@ -24,78 +23,79 @@ use deltachat::{config, provider};
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
 /// e.g. bitmask 7 triggers actions definded with bits 1, 2 and 4.
-fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
+async fn reset_tables(context: &Context, bits: i32) {
     println!("Resetting tables ({})...", bits);
     if 0 != bits & 1 {
-        sql::execute(context, &context.sql, "DELETE FROM jobs;", params![]).unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM jobs;", paramsx![])
+            .await
+            .unwrap();
         println!("(1) Jobs reset.");
     }
     if 0 != bits & 2 {
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM acpeerstates;",
-            params![],
-        )
-        .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM acpeerstates;", paramsx![])
+            .await
+            .unwrap();
         println!("(2) Peerstates reset.");
     }
     if 0 != bits & 4 {
-        sql::execute(context, &context.sql, "DELETE FROM keypairs;", params![]).unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM keypairs;", paramsx![])
+            .await
+            .unwrap();
         println!("(4) Private keypairs reset.");
     }
     if 0 != bits & 8 {
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM contacts WHERE id>9;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM chats WHERE id>9;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM chats_contacts;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM msgs WHERE id>9;",
-            params![],
-        )
-        .unwrap();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
-            params![],
-        )
-        .unwrap();
-        sql::execute(context, &context.sql, "DELETE FROM leftgrps;", params![]).unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM contacts WHERE id>9;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM chats WHERE id>9;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM chats_contacts;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM msgs WHERE id>9;", paramsx![])
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute(
+                "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
+                paramsx![],
+            )
+            .await
+            .unwrap();
+        context
+            .sql()
+            .execute("DELETE FROM leftgrps;", paramsx![])
+            .await
+            .unwrap();
         println!("(8) Rest but server config reset.");
     }
 
-    context.call_cb(Event::MsgsChanged {
+    context.emit_event(Event::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
-
-    1
 }
 
-fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
-    let data = dc_read_file(context, filename)?;
+async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
+    let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false) {
+    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -104,38 +104,29 @@ fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(),
 /// Import a file to the database.
 /// For testing, import a folder with eml-files, a single eml-file, e-mail plus public key and so on.
 /// For normal importing, use imex().
-///
-/// @private @memberof Context
-/// @param context The context as created by dc_context_new().
-/// @param spec The file or directory to import. NULL for the last command.
-/// @return 1=success, 0=error.
-fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
-    if !context.sql.is_open() {
-        error!(context, "Import: Database not opened.");
-        return 0;
-    }
-
-    let mut read_cnt = 0;
+async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
+    let mut read_cnt: usize = 0;
 
     let real_spec: String;
 
-    /* if `spec` is given, remember it for later usage; if it is not given, try to use the last one */
+    // if `spec` is given, remember it for later usage; if it is not given, try to use the last one
     if let Some(spec) = spec {
         real_spec = spec.to_string();
         context
-            .sql
+            .sql()
             .set_raw_config(context, "import_spec", Some(&real_spec))
+            .await
             .unwrap();
     } else {
-        let rs = context.sql.get_raw_config(context, "import_spec");
+        let rs = context.sql().get_raw_config("import_spec").await;
         if rs.is_none() {
             error!(context, "Import: No file or folder given.");
-            return 0;
+            return false;
         }
         real_spec = rs.unwrap();
     }
     if let Some(suffix) = dc_get_filesuffix_lc(&real_spec) {
-        if suffix == "eml" && dc_poke_eml_file(context, &real_spec).is_ok() {
+        if suffix == "eml" && poke_eml_file(context, &real_spec).await.is_ok() {
             read_cnt += 1
         }
     } else {
@@ -144,7 +135,7 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
         let dir = std::fs::read_dir(dir_name);
         if dir.is_err() {
             error!(context, "Import: Cannot open directory \"{}\".", &real_spec,);
-            return 0;
+            return false;
         } else {
             let dir = dir.unwrap();
             for entry in dir {
@@ -157,7 +148,7 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
                     println!("Import: {}", path_plus_name);
-                    if dc_poke_eml_file(context, path_plus_name).is_ok() {
+                    if poke_eml_file(context, path_plus_name).await.is_ok() {
                         read_cnt += 1
                     }
                 }
@@ -166,16 +157,19 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.call_cb(Event::MsgsChanged {
+        context.emit_event(Event::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
     }
-    1
+    true
 }
 
-fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
-    let contact = Contact::get_by_id(context, msg.get_from_id()).expect("invalid contact");
+async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
+    let contact = Contact::get_by_id(context, msg.get_from_id())
+        .await
+        .expect("invalid contact");
+
     let contact_name = contact.get_name();
     let contact_id = contact.get_id();
 
@@ -218,7 +212,7 @@ fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     );
 }
 
-fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
+async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id.is_daymarker() {
@@ -234,8 +228,8 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
                 );
                 lines_out += 1
             }
-            let msg = Message::load_from_db(context, msg_id)?;
-            log_msg(context, "", &msg);
+            let msg = Message::load_from_db(context, msg_id).await?;
+            log_msg(context, "", &msg).await;
         }
     }
     if lines_out > 0 {
@@ -246,18 +240,18 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     Ok(())
 }
 
-fn log_contactlist(context: &Context, contacts: &Vec<u32>) {
-    let mut contacts = contacts.clone();
+async fn log_contactlist(context: &Context, contacts: &[u32]) {
+    let mut contacts = contacts.to_vec();
     if !contacts.contains(&1) {
         contacts.push(1);
     }
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, contact_id) {
+        if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
             let name = contact.get_name();
             let addr = contact.get_addr();
-            let verified_state = contact.is_verified(context);
+            let verified_state = contact.is_verified(context).await;
             let verified_str = if VerifiedStatus::Unverified != verified_state {
                 if verified_state == VerifiedStatus::BidirectVerified {
                     " √√"
@@ -281,8 +275,8 @@ fn log_contactlist(context: &Context, contacts: &Vec<u32>) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, &context.sql, &addr);
-            if peerstate.is_some() && contact_id != 1 as libc::c_uint {
+            let peerstate = Peerstate::from_addr(context, &addr).await;
+            if peerstate.is_ok() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
                     peerstate.as_ref().unwrap().prefer_encrypt
@@ -298,10 +292,9 @@ fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
-    let chat_id = *context.cmdline_sel_chat_id.read().unwrap();
+pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<(), Error> {
     let mut sel_chat = if !chat_id.is_unset() {
-        Chat::load_from_db(context, chat_id).ok()
+        Chat::load_from_db(&context, *chat_id).await.ok()
     } else {
         None
     };
@@ -342,7 +335,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  configure\n\
                  connect\n\
                  disconnect\n\
-                 interrupt\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -405,7 +397,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  ============================================="
             ),
         },
-        "initiate-key-transfer" => match initiate_key_transfer(context) {
+        "initiate-key-transfer" => match initiate_key_transfer(&context).await {
             Ok(setup_code) => println!(
                 "Setup code for the transferred setup message: {}",
                 setup_code,
@@ -415,9 +407,9 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "get-setupcodebegin" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let msg_id: MsgId = MsgId::new(arg1.parse()?);
-            let msg = Message::load_from_db(context, msg_id)?;
+            let msg = Message::load_from_db(&context, msg_id).await?;
             if msg.is_setupmessage() {
-                let setupcodebegin = msg.get_setupcodebegin(context);
+                let setupcodebegin = msg.get_setupcodebegin(&context).await;
                 println!(
                     "The setup code for setup message {} starts with: {}",
                     msg_id,
@@ -432,29 +424,29 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(context, MsgId::new(arg1.parse()?), &arg2)?;
+            continue_key_transfer(&context, MsgId::new(arg1.parse()?), &arg2).await?;
         }
         "has-backup" => {
-            has_backup(context, blobdir)?;
+            has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            imex(context, ImexMode::ExportBackup, Some(blobdir));
+            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(context, ImexMode::ImportBackup, Some(arg1));
+            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
-            imex(context, ImexMode::ExportSelfKeys, Some(blobdir));
+            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
         }
         "import-keys" => {
-            imex(context, ImexMode::ImportSelfKeys, Some(blobdir));
+            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
         }
         "export-setup" => {
-            let setup_code = create_setup_code(context);
+            let setup_code = create_setup_code(&context);
             let file_name = blobdir.join("autocrypt-setup-message.html");
-            let file_content = render_setup_file(context, &setup_code)?;
-            std::fs::write(&file_name, file_content)?;
+            let file_content = render_setup_file(&context, &setup_code).await?;
+            async_std::fs::write(&file_name, file_content).await?;
             println!(
                 "Setup message written to: {}\nSetup code: {}",
                 file_name.display(),
@@ -462,49 +454,51 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             );
         }
         "poke" => {
-            ensure!(0 != poke_spec(context, Some(arg1)), "Poke failed");
+            ensure!(poke_spec(&context, Some(arg1)).await, "Poke failed");
         }
         "reset" => {
             ensure!(!arg1.is_empty(), "Argument <bits> missing: 1=jobs, 2=peerstates, 4=private keys, 8=rest but server config");
             let bits: i32 = arg1.parse()?;
             ensure!(bits < 16, "<bits> must be lower than 16.");
-            ensure!(0 != dc_reset_tables(context, bits), "Reset failed");
+            reset_tables(&context, bits).await;
         }
         "stop" => {
-            context.stop_ongoing();
+            context.stop_ongoing().await;
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
             let key = config::Config::from_str(&arg1)?;
             let value = if arg2.is_empty() { None } else { Some(arg2) };
-            context.set_config(key, value)?;
+            context.set_config(key, value).await?;
         }
         "get" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
             let key = config::Config::from_str(&arg1)?;
-            let val = context.get_config(key);
+            let val = context.get_config(key).await;
             println!("{}={:?}", key, val);
         }
         "info" => {
-            println!("{:#?}", context.get_info());
-        }
-        "interrupt" => {
-            interrupt_inbox_idle(context);
+            println!("{:#?}", context.get_info().await);
         }
         "maybenetwork" => {
-            maybe_network(context);
+            context.maybe_network().await;
         }
         "housekeeping" => {
-            sql::housekeeping(context);
+            sql::housekeeping(&context).await?;
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
+            let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
-                context,
+                &context,
                 listflags,
                 if arg1.is_empty() { None } else { Some(arg1) },
                 None,
-            )?;
+            )
+            .await?;
+            let time_needed = std::time::SystemTime::now()
+                .duration_since(time_start)
+                .unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -513,20 +507,20 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 );
 
                 for i in (0..cnt).rev() {
-                    let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
                         "{}#{}: {} [{} fresh] {}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat.get_id().get_fresh_msg_cnt(context),
+                        chat.get_id().get_fresh_msg_cnt(&context).await,
                         match chat.visibility {
                             ChatVisibility::Normal => "",
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
                     );
-                    let lot = chatlist.get_summary(context, i, Some(&chat));
+                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
@@ -559,31 +553,32 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(context, ChatId::new(0)) {
+            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
+            println!("{:?} to create this list", time_needed);
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
                 bail!("Argument [chat-id] is missing.");
             }
             if !arg1.is_empty() {
-                let chat_id = ChatId::new(arg1.parse()?);
-                println!("Selecting chat {}", chat_id);
-                sel_chat = Some(Chat::load_from_db(context, chat_id)?);
-                *context.cmdline_sel_chat_id.write().unwrap() = chat_id;
+                let id = ChatId::new(arg1.parse()?);
+                println!("Selecting chat {}", id);
+                sel_chat = Some(Chat::load_from_db(&context, id).await?);
+                *chat_id = id;
             }
 
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, None);
-            let members = chat::get_chat_contacts(context, sel_chat.id);
+            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await?;
+            let members = chat::get_chat_contacts(&context, sel_chat.id).await?;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
             } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
-                let contact = Contact::get_by_id(context, members[0])?;
+                let contact = Contact::get_by_id(&context, members[0]).await?;
                 contact.get_addr().to_string()
             } else {
                 format!("{} member(s)", members.len())
@@ -599,7 +594,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 } else {
                     ""
                 },
-                match sel_chat.get_profile_image(context) {
+                match sel_chat.get_profile_image(&context).await? {
                     Some(icon) => match icon.to_str() {
                         Some(icon) => format!(" Icon: {}", icon),
                         _ => " Icon: Err".to_string(),
@@ -607,38 +602,42 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     _ => "".to_string(),
                 },
             );
-            log_msglist(context, &msglist)?;
-            if let Some(draft) = sel_chat.get_id().get_draft(context)? {
-                log_msg(context, "Draft", &draft);
+            log_msglist(&context, &msglist).await?;
+            if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
+                log_msg(&context, "Draft", &draft).await;
             }
 
-            println!("{} messages.", sel_chat.get_id().get_msg_cnt(context));
-            chat::marknoticed_chat(context, sel_chat.get_id())?;
+            println!(
+                "{} messages.",
+                sel_chat.get_id().get_msg_cnt(&context).await
+            );
+            chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id: libc::c_int = arg1.parse()?;
-            let chat_id = chat::create_by_contact_id(context, contact_id as u32)?;
+            let chat_id = chat::create_by_contact_id(&context, contact_id as u32).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
         "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
             let msg_id = MsgId::new(arg1.parse()?);
-            let chat_id = chat::create_by_msg_id(context, msg_id)?;
-            let chat = Chat::load_from_db(context, chat_id)?;
+            let chat_id = chat::create_by_msg_id(&context, msg_id).await?;
+            let chat = Chat::load_from_db(&context, chat_id).await?;
 
             println!("{}#{} created successfully.", chat_prefix(&chat), chat_id,);
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(context, VerifiedStatus::Unverified, arg1)?;
+            let chat_id =
+                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
         "createverified" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(context, VerifiedStatus::Verified, arg1)?;
+            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
 
             println!("VerifiedGroup#{} created successfully.", chat_id);
         }
@@ -648,10 +647,12 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             let contact_id_0: libc::c_int = arg1.parse()?;
             if chat::add_contact_to_chat(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 contact_id_0 as u32,
-            ) {
+            )
+            .await
+            {
                 println!("Contact added to chat.");
             } else {
                 bail!("Cannot add contact to chat.");
@@ -662,17 +663,18 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id_1: libc::c_int = arg1.parse()?;
             chat::remove_contact_from_chat(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 contact_id_1 as u32,
-            )?;
+            )
+            .await?;
 
             println!("Contact added to chat.");
         }
         "groupname" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            chat::set_chat_name(context, sel_chat.as_ref().unwrap().get_id(), arg1)?;
+            chat::set_chat_name(&context, sel_chat.as_ref().unwrap().get_id(), arg1).await?;
 
             println!("Chat name set");
         }
@@ -680,24 +682,27 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <image> missing.");
 
-            chat::set_chat_profile_image(context, sel_chat.as_ref().unwrap().get_id(), arg1)?;
+            chat::set_chat_profile_image(&context, sel_chat.as_ref().unwrap().get_id(), arg1)
+                .await?;
 
             println!("Chat image set");
         }
         "chatinfo" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contacts = chat::get_chat_contacts(context, sel_chat.as_ref().unwrap().get_id());
+            let contacts =
+                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
             println!("Memberlist:");
 
-            log_contactlist(context, &contacts);
+            log_contactlist(&context, &contacts).await;
             println!(
                 "{} contacts\nLocation streaming: {}",
                 contacts.len(),
                 location::is_sending_locations_to_chat(
-                    context,
+                    &context,
                     sel_chat.as_ref().unwrap().get_id()
-                ),
+                )
+                .await,
             );
         }
         "getlocations" => {
@@ -705,12 +710,13 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             let contact_id = arg1.parse().unwrap_or_default();
             let locations = location::get_range(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 contact_id,
                 0,
                 0,
-            );
+            )
+            .await;
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
@@ -736,7 +742,12 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             ensure!(!arg1.is_empty(), "No timeout given.");
 
             let seconds = arg1.parse()?;
-            location::send_locations_to_chat(context, sel_chat.as_ref().unwrap().get_id(), seconds);
+            location::send_locations_to_chat(
+                &context,
+                sel_chat.as_ref().unwrap().get_id(),
+                seconds,
+            )
+            .await;
             println!(
                 "Locations will be sent to Chat#{} for {} seconds. Use 'setlocation <lat> <lng>' to play around.",
                 sel_chat.as_ref().unwrap().get_id(),
@@ -751,7 +762,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             let latitude = arg1.parse()?;
             let longitude = arg2.parse()?;
 
-            let continue_streaming = location::set(context, latitude, longitude, 0.);
+            let continue_streaming = location::set(&context, latitude, longitude, 0.).await?;
             if continue_streaming {
                 println!("Success, streaming should be continued.");
             } else {
@@ -759,7 +770,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             }
         }
         "dellocations" => {
-            location::delete_all(context)?;
+            location::delete_all(&context).await?;
         }
         "send" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -767,11 +778,11 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             let msg = format!("{} {}", arg1, arg2);
 
-            chat::send_text_msg(context, sel_chat.as_ref().unwrap().get_id(), msg)?;
+            chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), msg).await?;
         }
         "sendempty" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
-            chat::send_text_msg(context, sel_chat.as_ref().unwrap().get_id(), "".into())?;
+            chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), "".into()).await?;
         }
         "sendimage" | "sendfile" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -786,7 +797,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             if !arg2.is_empty() {
                 msg.set_text(Some(arg2.to_string()));
             }
-            chat::send_msg(context, sel_chat.as_ref().unwrap().get_id(), &mut msg)?;
+            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
@@ -797,9 +808,9 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 ChatId::new(0)
             };
 
-            let msglist = context.search_msgs(chat, arg1);
+            let msglist = context.search_msgs(chat, arg1).await;
 
-            log_msglist(context, &msglist)?;
+            log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
         }
         "draft" => {
@@ -812,10 +823,16 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     .as_ref()
                     .unwrap()
                     .get_id()
-                    .set_draft(context, Some(&mut draft));
+                    .set_draft(&context, Some(&mut draft))
+                    .await;
                 println!("Draft saved.");
             } else {
-                sel_chat.as_ref().unwrap().get_id().set_draft(context, None);
+                sel_chat
+                    .as_ref()
+                    .unwrap()
+                    .get_id()
+                    .set_draft(&context, None)
+                    .await;
                 println!("Draft deleted.");
             }
         }
@@ -826,21 +843,22 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             );
             let mut msg = Message::new(Viewtype::Text);
             msg.set_text(Some(arg1.to_string()));
-            chat::add_device_msg(context, None, Some(&mut msg))?;
+            chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
         "updatedevicechats" => {
-            context.update_device_chats()?;
+            context.update_device_chats().await?;
         }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let images = chat::get_chat_media(
-                context,
+                &context,
                 sel_chat.as_ref().unwrap().get_id(),
                 Viewtype::Image,
                 Viewtype::Gif,
                 Viewtype::Video,
-            );
+            )
+            .await?;
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
@@ -849,36 +867,38 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     print!(", {}", data);
                 }
             }
-            print!("\n");
+            println!();
         }
         "archive" | "unarchive" | "pin" | "unpin" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.set_visibility(
-                context,
-                match arg0 {
-                    "archive" => ChatVisibility::Archived,
-                    "unarchive" | "unpin" => ChatVisibility::Normal,
-                    "pin" => ChatVisibility::Pinned,
-                    _ => panic!("Unexpected command (This should never happen)"),
-                },
-            )?;
+            chat_id
+                .set_visibility(
+                    &context,
+                    match arg0 {
+                        "archive" => ChatVisibility::Archived,
+                        "unarchive" | "unpin" => ChatVisibility::Normal,
+                        "pin" => ChatVisibility::Pinned,
+                        _ => panic!("Unexpected command (This should never happen)"),
+                    },
+                )
+                .await?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.delete(context)?;
+            chat_id.delete(&context).await?;
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
-            let res = message::get_msg_info(context, id);
+            let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
         "listfresh" => {
-            let msglist = context.get_fresh_msgs();
+            let msglist = context.get_fresh_msgs().await;
 
-            log_msglist(context, &msglist)?;
+            log_msglist(&context, &msglist).await?;
             print!("{} fresh messages.", msglist.len());
         }
         "forward" => {
@@ -890,37 +910,38 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             let mut msg_ids = [MsgId::new(0); 1];
             let chat_id = ChatId::new(arg2.parse()?);
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            chat::forward_msgs(context, &msg_ids, chat_id)?;
+            chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [MsgId::new(0); 1];
+            let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(context, &msg_ids);
+            message::markseen_msgs(&context, msg_ids).await;
         }
         "star" | "unstar" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [MsgId::new(0); 1];
+            let mut msg_ids = vec![MsgId::new(0); 1];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::star_msgs(context, &msg_ids, arg0 == "star");
+            message::star_msgs(&context, msg_ids, arg0 == "star").await;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
             ids[0] = MsgId::new(arg1.parse()?);
-            message::delete_msgs(context, &ids);
+            message::delete_msgs(&context, &ids).await;
         }
         "listcontacts" | "contacts" | "listverified" => {
             let contacts = Contact::get_all(
-                context,
+                &context,
                 if arg0 == "listverified" {
                     0x1 | 0x2
                 } else {
                     0x2
                 },
                 Some(arg1),
-            )?;
-            log_contactlist(context, &contacts);
+            )
+            .await?;
+            log_contactlist(&context, &contacts).await;
             println!("{} contacts.", contacts.len());
         }
         "addcontact" => {
@@ -928,30 +949,30 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(context, book)?;
+                Contact::add_address_book(&context, book).await?;
             } else {
-                Contact::create(context, "", arg1)?;
+                Contact::create(&context, "", arg1).await?;
             }
         }
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
             let contact_id = arg1.parse()?;
-            let contact = Contact::get_by_id(context, contact_id)?;
+            let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
             let mut res = format!(
                 "Contact info for: {}:\nIcon: {}\n",
                 name_n_addr,
-                match contact.get_profile_image(context) {
+                match contact.get_profile_image(&context).await {
                     Some(image) => image.to_str().unwrap().to_string(),
                     None => "NoIcon".to_string(),
                 }
             );
 
-            res += &Contact::get_encrinfo(context, contact_id)?;
+            res += &Contact::get_encrinfo(&context, contact_id).await?;
 
-            let chatlist = Chatlist::try_load(context, 0, None, Some(contact_id))?;
+            let chatlist = Chatlist::try_load(&context, 0, None, Some(contact_id)).await?;
             let chatlist_cnt = chatlist.len();
             if chatlist_cnt > 0 {
                 res += &format!(
@@ -962,7 +983,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                     if 0 != i {
                         res += ", ";
                     }
-                    let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     res += &format!("{}#{}", chat_prefix(&chat), chat.get_id());
                 }
             }
@@ -971,11 +992,11 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            Contact::delete(context, arg1.parse()?)?;
+            Contact::delete(&context, arg1.parse()?).await?;
         }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            let res = check_qr(context, arg1);
+            let res = check_qr(&context, arg1).await;
             println!(
                 "state={}, id={}, text1={:?}, text2={:?}",
                 res.get_state(),
@@ -986,7 +1007,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            match set_config_from_qr(context, arg1) {
+            match set_config_from_qr(&context, arg1).await {
                 Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
                 Err(err) => println!("Cannot set config from QR code: {:?}", err),
             }
@@ -1014,7 +1035,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
         //     let event = arg1.parse()?;
         //     let event = Event::from_u32(event).ok_or(format_err!("Event::from_u32({})", event))?;
-        //     let r = context.call_cb(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
+        //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",
         //         event, event as usize, r as libc::c_int,
@@ -1023,7 +1044,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 
-            if let Ok(buf) = dc_read_file(context, &arg1) {
+            if let Ok(buf) = dc_read_file(&context, &arg1).await {
                 let (width, height) = dc_get_filemeta(&buf)?;
                 println!("width={}, height={}", width, height);
             } else {
@@ -1033,8 +1054,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "estimatedeletion" => {
             ensure!(!arg1.is_empty(), "Argument <seconds> missing");
             let seconds = arg1.parse()?;
-            let device_cnt = message::estimate_deletion_cnt(context, false, seconds)?;
-            let server_cnt = message::estimate_deletion_cnt(context, true, seconds)?;
+            let device_cnt = message::estimate_deletion_cnt(&context, false, seconds).await?;
+            let server_cnt = message::estimate_deletion_cnt(&context, true, seconds).await?;
             println!(
                 "estimated count of messages older than {} seconds:\non device: {}\non server: {}",
                 seconds, device_cnt, server_cnt
@@ -1043,7 +1064,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         "emptyserver" => {
             ensure!(!arg1.is_empty(), "Argument <flags> missing");
 
-            message::dc_empty_server(context, arg1.parse()?);
+            message::dc_empty_server(&context, arg1.parse()?).await;
         }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),

examples/repl/main.rs

@@ -6,26 +6,21 @@
 
 #[macro_use]
 extern crate deltachat;
-#[macro_use]
-extern crate lazy_static;
-#[macro_use]
-extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
-use std::path::Path;
 use std::process::Command;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
 
+use ansi_term::Color;
 use anyhow::{bail, Error};
+use async_std::path::Path;
 use deltachat::chat::ChatId;
 use deltachat::config;
 use deltachat::context::*;
-use deltachat::job::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
 use deltachat::Event;
+use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
@@ -38,179 +33,83 @@ use rustyline::{
 mod cmdline;
 use self::cmdline::*;
 
-// Event Handler
-
-fn receive_event(_context: &Context, event: Event) {
+/// Event Handler
+fn receive_event(event: Event) {
+    let yellow = Color::Yellow.normal();
     match event {
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
-            println!("{}", msg);
+            info!("{}", msg);
         }
         Event::SmtpConnected(msg) => {
-            println!("[DC_EVENT_SMTP_CONNECTED] {}", msg);
+            info!("[SMTP_CONNECTED] {}", msg);
         }
         Event::ImapConnected(msg) => {
-            println!("[DC_EVENT_IMAP_CONNECTED] {}", msg);
+            info!("[IMAP_CONNECTED] {}", msg);
         }
         Event::SmtpMessageSent(msg) => {
-            println!("[DC_EVENT_SMTP_MESSAGE_SENT] {}", msg);
+            info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
         Event::Warning(msg) => {
-            println!("[Warning] {}", msg);
+            warn!("{}", msg);
         }
         Event::Error(msg) => {
-            println!("\x1b[31m[DC_EVENT_ERROR] {}\x1b[0m", msg);
+            error!("{}", msg);
         }
         Event::ErrorNetwork(msg) => {
-            println!("\x1b[31m[DC_EVENT_ERROR_NETWORK] msg={}\x1b[0m", msg);
+            error!("[NETWORK] msg={}", msg);
         }
         Event::ErrorSelfNotInGroup(msg) => {
-            println!("\x1b[31m[DC_EVENT_ERROR_SELF_NOT_IN_GROUP] {}\x1b[0m", msg);
+            error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
         Event::MsgsChanged { chat_id, msg_id } => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_MSGS_CHANGED(chat_id={}, msg_id={})}}\n\x1b[0m",
-                chat_id, msg_id,
+            info!(
+                "{}",
+                yellow.paint(format!(
+                    "Received MSGS_CHANGED(chat_id={}, msg_id={})",
+                    chat_id, msg_id,
+                ))
             );
         }
         Event::ContactsChanged(_) => {
-            print!("\x1b[33m{{Received DC_EVENT_CONTACTS_CHANGED()}}\n\x1b[0m");
+            info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
         Event::LocationChanged(contact) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_LOCATION_CHANGED(contact={:?})}}\n\x1b[0m",
-                contact,
+            info!(
+                "{}",
+                yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
         Event::ConfigureProgress(progress) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_CONFIGURE_PROGRESS({} ‰)}}\n\x1b[0m",
-                progress,
+            info!(
+                "{}",
+                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
             );
         }
         Event::ImexProgress(progress) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_IMEX_PROGRESS({} ‰)}}\n\x1b[0m",
-                progress,
+            info!(
+                "{}",
+                yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
         Event::ImexFileWritten(file) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_IMEX_FILE_WRITTEN({})}}\n\x1b[0m",
-                file.display()
+            info!(
+                "{}",
+                yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
         Event::ChatModified(chat) => {
-            print!(
-                "\x1b[33m{{Received DC_EVENT_CHAT_MODIFIED({})}}\n\x1b[0m",
-                chat
+            info!(
+                "{}",
+                yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
             );
         }
         _ => {
-            print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
+            info!("Received {:?}", event);
         }
     }
 }
 
-// Threads for waiting for messages and for jobs
-
-lazy_static! {
-    static ref HANDLE: Arc<Mutex<Option<Handle>>> = Arc::new(Mutex::new(None));
-    static ref IS_RUNNING: AtomicBool = AtomicBool::new(true);
-}
-
-struct Handle {
-    handle_imap: Option<std::thread::JoinHandle<()>>,
-    handle_mvbox: Option<std::thread::JoinHandle<()>>,
-    handle_sentbox: Option<std::thread::JoinHandle<()>>,
-    handle_smtp: Option<std::thread::JoinHandle<()>>,
-}
-
-macro_rules! while_running {
-    ($code:block) => {
-        if IS_RUNNING.load(Ordering::Relaxed) {
-            $code
-        } else {
-            break;
-        }
-    };
-}
-
-fn start_threads(c: Arc<RwLock<Context>>) {
-    if HANDLE.clone().lock().unwrap().is_some() {
-        return;
-    }
-
-    println!("Starting threads");
-    IS_RUNNING.store(true, Ordering::Relaxed);
-
-    let ctx = c.clone();
-    let handle_imap = std::thread::spawn(move || loop {
-        while_running!({
-            perform_inbox_jobs(&ctx.read().unwrap());
-            perform_inbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                let context = ctx.read().unwrap();
-                perform_inbox_idle(&context);
-            });
-        });
-    });
-
-    let ctx = c.clone();
-    let handle_mvbox = std::thread::spawn(move || loop {
-        while_running!({
-            perform_mvbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                perform_mvbox_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    let ctx = c.clone();
-    let handle_sentbox = std::thread::spawn(move || loop {
-        while_running!({
-            perform_sentbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                perform_sentbox_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    let ctx = c;
-    let handle_smtp = std::thread::spawn(move || loop {
-        while_running!({
-            perform_smtp_jobs(&ctx.read().unwrap());
-            while_running!({
-                perform_smtp_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    *HANDLE.clone().lock().unwrap() = Some(Handle {
-        handle_imap: Some(handle_imap),
-        handle_mvbox: Some(handle_mvbox),
-        handle_sentbox: Some(handle_sentbox),
-        handle_smtp: Some(handle_smtp),
-    });
-}
-
-fn stop_threads(context: &Context) {
-    if let Some(ref mut handle) = *HANDLE.clone().lock().unwrap() {
-        println!("Stopping threads");
-        IS_RUNNING.store(false, Ordering::Relaxed);
-
-        interrupt_inbox_idle(context);
-        interrupt_mvbox_idle(context);
-        interrupt_sentbox_idle(context);
-        interrupt_smtp_idle(context);
-
-        handle.handle_imap.take().unwrap().join().unwrap();
-        handle.handle_mvbox.take().unwrap().join().unwrap();
-        handle.handle_sentbox.take().unwrap().join().unwrap();
-        handle.handle_smtp.take().unwrap().join().unwrap();
-    }
-}
-
 // === The main loop
 
 struct DcHelper {
@@ -247,10 +146,8 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 11] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
-    "open",
-    "close",
     "set",
     "get",
     "oauth2",
@@ -369,69 +266,81 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-fn main_0(args: Vec<String>) -> Result<(), Error> {
+async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(
-        Box::new(receive_event),
-        "CLI".into(),
-        Path::new(&args[1]).to_path_buf(),
-    )?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
+
+    let events = context.get_event_emitter();
+    async_std::task::spawn(async move {
+        while let Some(event) = events.recv().await {
+            receive_event(event);
+        }
+    });
 
     println!("Delta Chat Core is awaiting your commands.");
 
-    let ctx = Arc::new(RwLock::new(context));
-
     let config = Config::builder()
         .history_ignore_space(true)
         .completion_type(CompletionType::List)
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let h = DcHelper {
-        completer: FilenameCompleter::new(),
-        highlighter: MatchingBracketHighlighter::new(),
-        hinter: HistoryHinter {},
-    };
-    let mut rl = Editor::with_config(config);
-    rl.set_helper(Some(h));
-    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
-    if rl.load_history(".dc-history.txt").is_err() {
-        println!("No previous history.");
-    }
+    let mut selected_chat = ChatId::default();
+    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let input_loop = async_std::task::spawn_blocking(move || {
+        let h = DcHelper {
+            completer: FilenameCompleter::new(),
+            highlighter: MatchingBracketHighlighter::new(),
+            hinter: HistoryHinter {},
+        };
+        let mut rl = Editor::with_config(config);
+        rl.set_helper(Some(h));
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        if rl.load_history(".dc-history.txt").is_err() {
+            println!("No previous history.");
+        }
 
-    loop {
-        let p = "> ";
-        let readline = rl.readline(&p);
-        match readline {
-            Ok(line) => {
-                // TODO: ignore "set mail_pw"
-                rl.add_history_entry(line.as_str());
-                let ctx = ctx.clone();
-                match handle_cmd(line.trim(), ctx) {
-                    Ok(ExitResult::Continue) => {}
-                    Ok(ExitResult::Exit) => break,
-                    Err(err) => println!("Error: {}", err),
+        loop {
+            let p = "> ";
+            let readline = rl.readline(&p);
+
+            match readline {
+                Ok(line) => {
+                    // TODO: ignore "set mail_pw"
+                    rl.add_history_entry(line.as_str());
+                    async_std::task::block_on(reader_s.send(line));
+                }
+                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                    println!("Exiting...");
+                    drop(reader_s);
+                    break;
+                }
+                Err(err) => {
+                    println!("Error: {}", err);
+                    drop(reader_s);
+                    break;
                 }
             }
-            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-                println!("Exiting...");
-                break;
-            }
-            Err(err) => {
-                println!("Error: {}", err);
-                break;
-            }
+        }
+
+        rl.save_history(".dc-history.txt")?;
+        println!("history saved");
+        Ok::<_, Error>(())
+    });
+
+    while let Ok(line) = reader_r.recv().await {
+        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
+            Ok(ExitResult::Continue) => {}
+            Ok(ExitResult::Exit) => break,
+            Err(err) => println!("Error: {}", err),
         }
     }
-    rl.save_history(".dc-history.txt")?;
-    println!("history saved");
-    {
-        stop_threads(&ctx.read().unwrap());
-    }
+    context.stop_io().await;
+    input_loop.await?;
 
     Ok(())
 }
@@ -442,43 +351,29 @@ enum ExitResult {
     Exit,
 }
 
-fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
+async fn handle_cmd(
+    line: &str,
+    ctx: Context,
+    selected_chat: &mut ChatId,
+) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
 
     match arg0 {
         "connect" => {
-            start_threads(ctx);
+            ctx.start_io().await;
         }
         "disconnect" => {
-            stop_threads(&ctx.read().unwrap());
-        }
-        "smtp-jobs" => {
-            if HANDLE.clone().lock().unwrap().is_some() {
-                println!("smtp-jobs are already running in a thread.",);
-            } else {
-                perform_smtp_jobs(&ctx.read().unwrap());
-            }
-        }
-        "imap-jobs" => {
-            if HANDLE.clone().lock().unwrap().is_some() {
-                println!("inbox-jobs are already running in a thread.");
-            } else {
-                perform_inbox_jobs(&ctx.read().unwrap());
-            }
+            ctx.stop_io().await;
         }
         "configure" => {
-            start_threads(ctx.clone());
-            ctx.read().unwrap().configure();
+            ctx.configure().await?;
         }
         "oauth2" => {
-            if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
-                let oauth2_url = dc_get_oauth2_url(
-                    &ctx.read().unwrap(),
-                    &addr,
-                    "chat.delta:/com.b44t.messenger",
-                );
+            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
+                let oauth2_url =
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -493,11 +388,10 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
             print!("\x1b[1;1H\x1b[2J");
         }
         "getqr" | "getbadqr" => {
-            start_threads(ctx.clone());
-            if let Some(mut qr) = dc_get_securejoin_qr(
-                &ctx.read().unwrap(),
-                ChatId::new(arg1.parse().unwrap_or_default()),
-            ) {
+            ctx.start_io().await;
+            if let Some(mut qr) =
+                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
+            {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -513,23 +407,23 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
             }
         }
         "joinqr" => {
-            start_threads(ctx.clone());
+            ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx.read().unwrap(), arg1);
+                dc_join_securejoin(&ctx, arg1).await;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),
-        _ => dc_cmdline(&ctx.read().unwrap(), line)?,
+        _ => cmdline(ctx.clone(), line, selected_chat).await?,
     }
 
     Ok(ExitResult::Continue)
 }
 
-pub fn main() -> Result<(), Error> {
+fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
-    let args: Vec<String> = std::env::args().collect();
-    main_0(args)?;
+    let args = std::env::args().collect();
+    async_std::task::block_on(async move { start(args).await })?;
 
     Ok(())
 }

examples/simple.rs

@@ -1,7 +1,3 @@
-extern crate deltachat;
-
-use std::sync::{Arc, RwLock};
-use std::{thread, time};
 use tempfile::tempdir;
 
 use deltachat::chat;
@@ -9,103 +5,96 @@ use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::job::{
-    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
-    perform_smtp_jobs,
-};
+use deltachat::message::Message;
 use deltachat::Event;
 
-fn cb(_ctx: &Context, event: Event) {
-    print!("[{:?}]", event);
-
+fn cb(event: Event) {
     match event {
         Event::ConfigureProgress(progress) => {
-            println!("  progress: {}", progress);
+            log::info!("progress: {}", progress);
         }
-        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            println!("  {}", msg);
+        Event::Info(msg) => {
+            log::info!("{}", msg);
         }
-        _ => {
-            println!();
+        Event::Warning(msg) => {
+            log::warn!("{}", msg);
+        }
+        Event::Error(msg) | Event::ErrorNetwork(msg) => {
+            log::error!("{}", msg);
+        }
+        event => {
+            log::info!("{:?}", event);
         }
     }
 }
 
-fn main() {
+/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
+#[async_std::main]
+async fn main() {
+    pretty_env_logger::try_init_timed().ok();
+
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    println!("creating database {:?}", dbfile);
-    let ctx =
-        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
-    let running = Arc::new(RwLock::new(true));
-    let info = ctx.get_info();
-    let duration = time::Duration::from_millis(4000);
-    println!("info: {:#?}", info);
+    log::info!("creating database {:?}", dbfile);
+    let ctx = Context::new("FakeOs".into(), dbfile.into())
+        .await
+        .expect("Failed to create context");
+    let info = ctx.get_info().await;
+    log::info!("info: {:#?}", info);
 
-    let ctx = Arc::new(ctx);
-    let ctx1 = ctx.clone();
-    let r1 = running.clone();
-    let t1 = thread::spawn(move || {
-        while *r1.read().unwrap() {
-            perform_inbox_jobs(&ctx1);
-            if *r1.read().unwrap() {
-                perform_inbox_fetch(&ctx1);
-
-                if *r1.read().unwrap() {
-                    perform_inbox_idle(&ctx1);
-                }
-            }
+    let events = ctx.get_event_emitter();
+    let events_spawn = async_std::task::spawn(async move {
+        while let Some(event) = events.recv().await {
+            cb(event);
         }
     });
 
-    let ctx1 = ctx.clone();
-    let r1 = running.clone();
-    let t2 = thread::spawn(move || {
-        while *r1.read().unwrap() {
-            perform_smtp_jobs(&ctx1);
-            if *r1.read().unwrap() {
-                perform_smtp_idle(&ctx1);
-            }
-        }
-    });
-
-    println!("configuring");
+    log::info!("configuring");
     let args = std::env::args().collect::<Vec<String>>();
-    assert_eq!(args.len(), 2, "missing password");
-    let pw = args[1].clone();
-    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
+    assert_eq!(args.len(), 3, "requires email password");
+    let email = args[1].clone();
+    let pw = args[2].clone();
+    ctx.set_config(config::Config::Addr, Some(&email))
+        .await
+        .unwrap();
+    ctx.set_config(config::Config::MailPw, Some(&pw))
+        .await
         .unwrap();
-    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-    ctx.configure();
 
-    thread::sleep(duration);
+    ctx.configure().await.unwrap();
 
-    println!("sending a message");
-    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
-    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
-    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
+    log::info!("------ RUN ------");
+    ctx.start_io().await;
+    log::info!("--- SENDING A MESSAGE ---");
 
-    println!("fetching chats..");
-    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
+    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
+        .await
+        .unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
-    for i in 0..chats.len() {
-        let summary = chats.get_summary(&ctx, 0, None);
-        let text1 = summary.get_text1();
-        let text2 = summary.get_text2();
-        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
+    for i in 0..1 {
+        log::info!("sending message {}", i);
+        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
+            .await
+            .unwrap();
     }
 
-    thread::sleep(duration);
+    // wait for the message to be sent out
+    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
 
-    println!("stopping threads");
+    log::info!("fetching chats..");
+    let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
-    *running.write().unwrap() = false;
-    deltachat::job::interrupt_inbox_idle(&ctx);
-    deltachat::job::interrupt_smtp_idle(&ctx);
+    for i in 0..chats.len() {
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
+            .await
+            .unwrap();
+        log::info!("[{}] msg: {:?}", i, msg);
+    }
 
-    println!("joining");
-    t1.join().unwrap();
-    t2.join().unwrap();
-
-    println!("closing");
+    log::info!("stopping");
+    ctx.stop_io().await;
+    log::info!("closing");
+    drop(ctx);
+    events_spawn.await;
 }

python/README.rst

@@ -113,10 +113,10 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux1 based wheels
-================================
+Building manylinux based wheels
+====================================
 
-Building portable manylinux1 wheels which come with libdeltachat.so
+Building portable manylinux wheels which come with libdeltachat.so
 can be done with docker-tooling.
 
 using docker pull / premade images

python/examples/echo_and_quit.py

@@ -12,10 +12,13 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+            if message.is_system_message():
+                message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))
+            else:
+                text = message.text
+                message.chat.send_text("echoing from {}:\n{}".format(addr, text))
 
     @account_hookimpl
     def ac_message_delivered(self, message):

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))

python/examples/test_examples.py

@@ -3,7 +3,7 @@ import pytest
 import py
 import echo_and_quit
 import group_tracking
-from deltachat.eventlogger import FFIEventLogger
+from deltachat.events import FFIEventLogger
 
 
 @pytest.fixture(scope='session')
@@ -17,17 +17,24 @@ def datadir():
         pytest.skip('test-data directory not found')
 
 
-def test_echo_quit_plugin(acfactory):
+def test_echo_quit_plugin(acfactory, lp):
+    lp.sec("creating one echo_and_quit bot")
     botproc = acfactory.run_bot_process(echo_and_quit)
 
+    lp.sec("creating a temp account to contact the bot")
     ac1 = acfactory.get_one_online_account()
+
+    lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
+    bot_chat = bot_contact.create_chat()
+    bot_chat.send_text("hello")
+
+    lp.sec("waiting for the reply message from the bot to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    assert reply.chat == ch1
-    ch1.send_text("/quit")
+    lp.sec("send quit sequence")
+    bot_chat.send_text("/quit")
     botproc.wait()
 
 
@@ -40,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+    ac1.add_account_plugin(FFIEventLogger(ac1))
+    ac2.add_account_plugin(FFIEventLogger(ac2))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -51,7 +58,7 @@ def test_group_tracking_plugin(acfactory, lp):
 
     botproc.fnmatch_lines("""
         *ac_chat_modified*bot test group*
-    """.format(ac1.get_config("addr")))
+    """)
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
     contact3 = ac1.create_contact(ac2.get_config("addr"))

python/fail_test.py

@@ -0,0 +1,7 @@
+from __future__ import print_function
+from deltachat import capi
+from deltachat.capi import ffi, lib
+
+if __name__ == "__main__":
+    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
+    lib.dc_stop_io(ctx)

python/install_python_bindings.py

@@ -19,6 +19,7 @@ if __name__ == "__main__":
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
     if target == 'release':
         cmd.append("--release")
+    print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -1,13 +1,13 @@
 import sys
 
-from . import capi, const, hookspec
-from .capi import ffi
+from . import capi, const, hookspec # noqa
+from .capi import ffi  # noqa
 from .account import Account  # noqa
 from .message import Message  # noqa
 from .contact import Contact  # noqa
 from .chat import Chat        # noqa
 from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import eventlogger
+from . import events
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -17,64 +17,6 @@ except DistributionNotFound:
     __version__ = "0.0.0.dev0-unknown"
 
 
-_DC_CALLBACK_MAP = {}
-
-
-@capi.ffi.def_extern()
-def py_dc_callback(ctx, evt, data1, data2):
-    """The global event handler.
-
-    CFFI only allows us to set one global event handler, so this one
-    looks up the correct event handler for the given context.
-    """
-    try:
-        callback = _DC_CALLBACK_MAP.get(ctx, lambda *a: 0)
-    except AttributeError:
-        # we are in a deep in GC-free/interpreter shutdown land
-        # nothing much better to do here than:
-        return 0
-
-    # the following code relates to the deltachat/_build.py's helper
-    # function which provides us signature info of an event call
-    evt_name = get_dc_event_name(evt)
-    event_sig_types = capi.lib.dc_get_event_signature_types(evt)
-    if data1 and event_sig_types & 1:
-        data1 = ffi.string(ffi.cast('char*', data1)).decode("utf8")
-    if data2 and event_sig_types & 2:
-        data2 = ffi.string(ffi.cast('char*', data2)).decode("utf8")
-        try:
-            if isinstance(data2, bytes):
-                data2 = data2.decode("utf8")
-        except UnicodeDecodeError:
-            # XXX ignoring the decode error is not quite correct but for now
-            # i don't want to hunt down encoding problems in the c lib
-            pass
-    try:
-        ret = callback(ctx, evt_name, data1, data2)
-        if ret is None:
-            ret = 0
-        assert isinstance(ret, int), repr(ret)
-        if event_sig_types & 4:
-            return ffi.cast('uintptr_t', ret)
-        elif event_sig_types & 8:
-            return ffi.cast('int', ret)
-    except:  # noqa
-        raise
-        ret = 0
-    return ret
-
-
-def set_context_callback(dc_context, func):
-    _DC_CALLBACK_MAP[dc_context] = func
-
-
-def clear_context_callback(dc_context):
-    try:
-        _DC_CALLBACK_MAP.pop(dc_context, None)
-    except AttributeError:
-        pass
-
-
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
         for name, val in vars(const).items():
@@ -97,7 +39,7 @@ def unregister_global_plugin(plugin):
     gm.unregister(plugin)
 
 
-register_global_plugin(eventlogger)
+register_global_plugin(events)
 
 
 def run_cmdline(argv=None, account_plugins=None):
@@ -118,9 +60,14 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = eventlogger.FFIEventLogger(ac, "bot")
+        ac.set_config("displayname", "bot")
+        log = events.FFIEventLogger(ac)
         ac.add_account_plugin(log)
 
+    for plugin in account_plugins or []:
+        print("adding plugin", plugin)
+        ac.add_account_plugin(plugin)
+
     if not ac.is_configured():
         assert args.email and args.password, (
             "you must specify --email and --password once to configure this database/account"
@@ -130,12 +77,11 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-
-    for plugin in account_plugins or []:
-        ac.add_account_plugin(plugin)
+        configtracker = ac.configure()
+        configtracker.wait_finish()
 
     # start IO threads and configure if neccessary
-    ac.start()
+    ac.start_io()
 
     print("{}: waiting for message".format(ac.get_config("addr")))
 

python/src/deltachat/_build.py

@@ -1,79 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            const char * dupstring_helper(const char* string)
-            {
-                return strdup(string);
-            }
-            int dc_get_event_signature_types(int e)
-            {
-                int result = 0;
-                if (DC_EVENT_DATA1_IS_STRING(e))
-                    result |= 1;
-                if (DC_EVENT_DATA2_IS_STRING(e))
-                    result |= 2;
-                if (DC_EVENT_RETURNS_STRING(e))
-                    result |= 4;
-                if (DC_EVENT_RETURNS_INT(e))
-                    result |= 8;
-                return result;
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern const char * dupstring_helper(const char* string);
-        extern int dc_get_event_signature_types(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -85,20 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
     builder.cdef("""
-        extern "Python" uintptr_t py_dc_callback(
-            dc_context_t* context,
-            int event,
-            uintptr_t data1,
-            uintptr_t data2);
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
     """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -1,22 +1,20 @@
 """ Account class implementation. """
 
 from __future__ import print_function
-import atexit
 from contextlib import contextmanager
 from email.utils import parseaddr
-import queue
 from threading import Event
 import os
 from array import array
-import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message, map_system_message
+from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker
-from . import hookspec, iothreads
+from .tracker import ImexTracker, ConfigureTracker
+from . import hookspec
+from .events import EventThread
 
 
 class MissingCredentials(ValueError):
@@ -30,7 +28,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None):
+    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -39,41 +37,41 @@ class Account(object):
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
+        self._logging = logging
+
         self.add_account_plugin(self)
 
-        self._dc_context = ffi.gc(
-            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
-            _destroy_dc_context,
-        )
-
-        hook = hookspec.Global._get_plugin_manager().hook
-
-        self._threads = iothreads.IOThreads(self)
-        self._hook_event_queue = queue.Queue()
-        self._in_use_iter_events = False
-        self._shutdown_event = Event()
-
-        # open database
         self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
-        if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
-            raise ValueError("Could not dc_open: {}".format(db_path))
+
+        self._dc_context = ffi.gc(
+            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
+            lib.dc_context_unref,
+        )
+        if self._dc_context == ffi.NULL:
+            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
+
+        self._shutdown_event = Event()
+        self._event_thread = EventThread(self)
         self._configkeys = self.get_config("sys.config_keys").split()
-        atexit.register(self.shutdown)
+        hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    @hookspec.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        for name, kwargs in self._map_ffi_event(ffi_event):
-            ev = HookEvent(self, name=name, kwargs=kwargs)
-            self._hook_event_queue.put(ev)
+    def disable_logging(self):
+        """ disable logging. """
+        self._logging = False
+
+    def enable_logging(self):
+        """ re-enable logging. """
+        self._logging = True
 
     # def __del__(self):
     #    self.shutdown()
 
-    def ac_log_line(self, msg):
-        self._pm.hook.ac_log_line(message=msg)
+    def log(self, msg):
+        if self._logging:
+            self._pm.hook.ac_log_line(message=msg)
 
     def _check_config_key(self, name):
         if name not in self._configkeys:
@@ -162,7 +160,7 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return bool(lib.dc_is_configured(self._dc_context))
+        return True if lib.dc_is_configured(self._dc_context) else False
 
     def set_avatar(self, img_path):
         """Set self avatar.
@@ -215,22 +213,40 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, email, name=None):
-        """ create a (new) Contact. If there already is a Contact
-        with that e-mail address, it is unblocked and its name is
-        updated.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        :param email: email-address (text type)
-        :param name: display name for this contact (optional)
+        Calling this method will always resulut in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        `name` is updated if specified.
+
+        :param obj: email-address, Account or Contact instance.
+        :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
+        if isinstance(obj, Account):
+            if not obj.is_configured():
+                raise ValueError("can only add addresses from configured accounts")
+            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
+        elif isinstance(obj, Contact):
+            if obj.account != self:
+                raise ValueError("account mismatch {}".format(obj))
+            addr, displayname = obj.addr, obj.name
+        elif isinstance(obj, str):
+            displayname, addr = parseaddr(obj)
+        else:
+            raise TypeError("don't know how to create chat for %r" % (obj, ))
+
+        if name is None and displayname:
+            name = displayname
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -240,7 +256,7 @@ class Account(object):
         :returns: True if deletion succeeded (contact was deleted)
         """
         contact_id = contact.id
-        assert contact._dc_context == self._dc_context
+        assert contact.account == self
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
@@ -252,6 +268,13 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -281,53 +304,29 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+    def create_chat(self, obj):
+        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        return self.create_contact(obj).create_chat()
 
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(contact, "id"):
-            if contact._dc_context != self._dc_context:
-                raise ValueError("Contact belongs to a different Account")
-            contact_id = contact.id
-        else:
-            assert isinstance(contact, int)
-            contact_id = contact
-        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
-        return Chat(self, chat_id)
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
 
-    def create_chat_by_message(self, message):
-        """ create or get an existing chat object for the
-        the specified message.
-
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
-        :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(message, "id"):
-            if self._dc_context != message._dc_context:
-                raise ValueError("Message belongs to a different Account")
-            msg_id = message.id
-        else:
-            assert isinstance(message, int)
-            msg_id = message
-        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
-        return Chat(self, chat_id)
-
-    def create_group_chat(self, name, verified=False):
+    def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
+        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        return Chat(self, chat_id)
+        chat = Chat(self, chat_id)
+        if contacts is not None:
+            for contact in contacts:
+                chat.add_contact(contact)
+        return chat
 
     def get_chats(self):
         """ return list of chats.
@@ -356,13 +355,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -428,8 +420,6 @@ class Account(object):
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -452,8 +442,6 @@ class Account(object):
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
             imex_tracker.wait_finish()
 
     def initiate_key_transfer(self):
@@ -462,8 +450,8 @@ class Account(object):
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
-        if not self._threads.is_started():
-            raise RuntimeError("threads not running, can not send out")
+        if not self.is_started():
+            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -545,6 +533,10 @@ class Account(object):
         self._pm.check_pending()
         return plugin
 
+    def remove_account_plugin(self, plugin, name=None):
+        """ remove an account plugin. """
+        self._pm.unregister(plugin, name=name)
+
     @contextmanager
     def temp_plugin(self, plugin):
         """ run a with-block with the given plugin temporarily registered. """
@@ -556,110 +548,77 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def start(self, callback_thread=True):
-        """ start this account (activate imap/smtp threads etc.)
-        and return immediately.
+    def start_io(self):
+        """ start this account's IO scheduling (Rust-core async scheduler)
 
-        If this account is not configured, an internal configuration
-        job will be scheduled if config values are sufficiently specified.
+        If this account is not configured an Exception is raised.
+        You need to call account.configure() and account.wait_configure_finish()
+        before.
 
-        You may call `wait_shutdown` or `shutdown` after the
-        account is in started mode.
+        You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
+        account is started.
 
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
+        :raises ConfigureFailed: if the account could not be configured.
 
         :returns: None
         """
         if not self.is_configured():
-            if not self.get_config("addr") or not self.get_config("mail_pw"):
-                raise MissingCredentials("addr or mail_pwd not set in config")
-            lib.dc_configure(self._dc_context)
-        self._threads.start(callback_thread=callback_thread)
+            raise ValueError("account not configured, cannot start io")
+        lib.dc_start_io(self._dc_context)
+
+    def configure(self):
+        """ Start configuration process and return a Configtracker instance
+        on which you can block with wait_finish() to get a True/False success
+        value for the configuration process.
+        """
+        assert not self.is_configured()
+        if not self.get_config("addr") or not self.get_config("mail_pw"):
+            raise MissingCredentials("addr or mail_pwd not set in config")
+        configtracker = ConfigureTracker(self)
+        self.add_account_plugin(configtracker)
+        lib.dc_configure(self._dc_context)
+        return configtracker
+
+    def is_started(self):
+        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
 
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def shutdown(self, wait=True):
-        """ shutdown account, stop threads and close and remove
-        underlying dc_context and callbacks. """
-        dc_context = self._dc_context
-        if dc_context is None:
+    def stop_io(self):
+        """ stop core IO scheduler if it is running. """
+        self.log("stop_ongoing")
+        self.stop_ongoing()
+
+        if bool(lib.dc_is_io_running(self._dc_context)):
+            self.log("dc_stop_io (stop core IO scheduler)")
+            lib.dc_stop_io(self._dc_context)
+        else:
+            self.log("stop_scheduler called on non-running context")
+
+    def shutdown(self):
+        """ shutdown and destroy account (stop callback thread, close and remove
+        underlying dc_context)."""
+        if self._dc_context is None:
             return
 
-        if self._threads.is_started():
-            self.stop_ongoing()
-            self._threads.stop(wait=False)
-        lib.dc_close(dc_context)
-        self._hook_event_queue.put(None)
-        self._threads.stop(wait=wait)  # to wait for threads
+        self.stop_io()
+
+        self.log("remove dc_context references")
+        # the dc_context_unref triggers get_next_event to return ffi.NULL
+        # which in turns makes the event thread finish execution
         self._dc_context = None
-        atexit.unregister(self.shutdown)
+
+        self.log("wait for event thread to finish")
+        self._event_thread.wait()
+
         self._shutdown_event.set()
+
         hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
-
-    def _handle_current_events(self):
-        """ handle all currently queued events and then return. """
-        while 1:
-            try:
-                event = self._hook_event_queue.get(block=False)
-            except queue.Empty:
-                break
-            else:
-                event.call_hook()
-
-    def iter_events(self, timeout=None):
-        """ yield hook events until shutdown.
-
-        It is not allowed to call iter_events() from multiple threads.
-        """
-        if self._in_use_iter_events:
-            raise RuntimeError("can only call iter_events() from one thread")
-        self._in_use_iter_events = True
-        while 1:
-            event = self._hook_event_queue.get(timeout=timeout)
-            if event is None:
-                break
-            yield event
-
-    def _map_ffi_event(self, ffi_event):
-        name = ffi_event.name
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-            data1 = ffi_event.data1
-            if data1 == 0 or data1 == 1000:
-                success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = self.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = self.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = self.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_CHAT_MODIFIED":
-            chat = self.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)
-
-
-def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
-    # destructor for dc_context
-    dc_context_unref(dc_context)
-    try:
-        deltachat.clear_context_callback(dc_context)
-    except (TypeError, AttributeError):
-        # we are deep into Python Interpreter shutdown,
-        # so no need to clear the callback context mapping.
-        pass
+        hook.dc_account_after_shutdown(account=self)
+        self.log("shutdown finished")
 
 
 class ScannedQRCode:
@@ -675,17 +634,3 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
-
-
-class HookEvent:
-    def __init__(self, account, name, kwargs):
-        assert hasattr(account._pm.hook, name), name
-        self.account = account
-        self.name = name
-        self.kwargs = kwargs
-
-    def call_hook(self):
-        hook = getattr(self.account._pm.hook, self.name, None)
-        if hook is None:
-            raise ValueError("event_name {} unknown".format(self.name))
-        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -18,13 +18,14 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
-        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
-               self._dc_context == getattr(other, "_dc_context", None)
+               self.account._dc_context == other.account._dc_context
 
     def __ne__(self, other):
         return not (self == other)
@@ -35,7 +36,7 @@ class Chat(object):
     @property
     def _dc_chat(self):
         return ffi.gc(
-            lib.dc_get_chat(self._dc_context, self.id),
+            lib.dc_get_chat(self.account._dc_context, self.id),
             lib.dc_chat_unref
         )
 
@@ -47,7 +48,7 @@ class Chat(object):
         - does not delete messages on server
         - the chat or contact is not blocked, new message will arrive
         """
-        lib.dc_delete_chat(self._dc_context, self.id)
+        lib.dc_delete_chat(self.account._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
@@ -105,7 +106,7 @@ class Chat(object):
         :returns: None
         """
         name = as_dc_charpointer(name)
-        return lib.dc_set_chat_name(self._dc_context, self.id, name)
+        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
 
     def mute(self, duration=None):
         """ mutes the chat
@@ -117,7 +118,7 @@ class Chat(object):
             mute_duration = -1
         else:
             mute_duration = duration
-        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, mute_duration)
+        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration)
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
@@ -126,7 +127,7 @@ class Chat(object):
 
         :returns: None
         """
-        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, 0)
+        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, 0)
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
@@ -152,7 +153,7 @@ class Chat(object):
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
         where account.join_with_qrcode(qr) needs to be called.
         """
-        res = lib.dc_get_securejoin_qr(self._dc_context, self.id)
+        res = lib.dc_get_securejoin_qr(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
     # ------  chat messaging API ------------------------------
@@ -174,7 +175,7 @@ class Chat(object):
             assert msg.id != 0
             # get a fresh copy of dc_msg, the core needs it
             msg = Message.from_db(self.account, msg.id)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         # modify message in place to avoid bad state for the caller
@@ -189,7 +190,7 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         msg = as_dc_charpointer(text)
-        msg_id = lib.dc_send_text_msg(self._dc_context, self.id, msg)
+        msg_id = lib.dc_send_text_msg(self.account._dc_context, self.id, msg)
         if msg_id == 0:
             raise ValueError("message could not be send, does chat exist?")
         return Message.from_db(self.account, msg_id)
@@ -204,7 +205,7 @@ class Chat(object):
         """
         msg = Message.new_empty(self.account, view_type="file")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
@@ -219,7 +220,7 @@ class Chat(object):
         mime_type = mimetypes.guess_type(path)[0]
         msg = Message.new_empty(self.account, view_type="image")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
@@ -230,7 +231,7 @@ class Chat(object):
         :param msg: the message to be prepared.
         :returns: :class:`deltachat.message.Message` instance.
         """
-        msg_id = lib.dc_prepare_msg(self._dc_context, self.id, msg._dc_msg)
+        msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
         # invalidate passed in message which is not safe to use anymore
@@ -266,7 +267,7 @@ class Chat(object):
         msg = Message.from_db(self.account, message.id)
 
         # pass 0 as chat-id because core-docs say it's ok when out-preparing
-        sent_id = lib.dc_send_msg(self._dc_context, 0, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self.account._dc_context, 0, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         assert sent_id == msg.id
@@ -280,9 +281,9 @@ class Chat(object):
         :returns: None
         """
         if message is None:
-            lib.dc_set_draft(self._dc_context, self.id, ffi.NULL)
+            lib.dc_set_draft(self.account._dc_context, self.id, ffi.NULL)
         else:
-            lib.dc_set_draft(self._dc_context, self.id, message._dc_msg)
+            lib.dc_set_draft(self.account._dc_context, self.id, message._dc_msg)
 
     def get_draft(self):
         """ get draft message for this chat.
@@ -290,7 +291,7 @@ class Chat(object):
         :param message: a :class:`Message` instance
         :returns: Message object or None (if no draft available)
         """
-        x = lib.dc_get_draft(self._dc_context, self.id)
+        x = lib.dc_get_draft(self.account._dc_context, self.id)
         if x == ffi.NULL:
             return None
         dc_msg = ffi.gc(x, lib.dc_msg_unref)
@@ -302,7 +303,7 @@ class Chat(object):
         :returns: list of :class:`deltachat.message.Message` objects for this chat.
         """
         dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(self._dc_context, self.id, 0, 0),
+            lib.dc_get_chat_msgs(self.account._dc_context, self.id, 0, 0),
             lib.dc_array_unref
         )
         return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
@@ -312,60 +313,69 @@ class Chat(object):
 
         :returns: number of fresh messages
         """
-        return lib.dc_get_fresh_msg_cnt(self._dc_context, self.id)
+        return lib.dc_get_fresh_msg_cnt(self.account._dc_context, self.id)
 
     def mark_noticed(self):
         """ mark all messages in this chat as noticed.
 
         Noticed messages are no longer fresh.
         """
-        return lib.dc_marknoticed_chat(self._dc_context, self.id)
+        return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
 
     def get_summary(self):
         """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
+        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
         s = from_dc_charpointer(dc_res)
         return json.loads(s)
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, contact):
+    def add_contact(self, obj):
         """ add a contact to this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
         :returns: None
         """
-        ret = lib.dc_add_contact_to_chat(self._dc_context, self.id, contact.id)
+        contact = self.account.create_contact(obj)
+        ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
+        return contact
 
-    def remove_contact(self, contact):
+    def remove_contact(self, obj):
         """ remove a contact from this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        ret = lib.dc_remove_contact_from_chat(self._dc_context, self.id, contact.id)
+        contact = self.account.create_contact(obj)
+        ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
-        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-
         """
         from .contact import Contact
         dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self._dc_context, self.id),
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
             lib.dc_array_unref
         )
         return list(iter_array(
             dc_array, lambda id: Contact(self.account, id))
         )
 
+    def num_contacts(self):
+        """ return number of contacts in this chat. """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return lib.dc_array_get_cnt(dc_array)
+
     def set_profile_image(self, img_path):
         """Set group profile image.
 
@@ -378,7 +388,7 @@ class Chat(object):
         """
         assert os.path.exists(img_path), img_path
         p = as_dc_charpointer(img_path)
-        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, p)
+        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, p)
         if res != 1:
             raise ValueError("Setting Profile Image {!r} failed".format(p))
 
@@ -391,7 +401,7 @@ class Chat(object):
         :raises ValueError: if profile image could not be reset
         :returns: None
         """
-        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, ffi.NULL)
+        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, ffi.NULL)
         if res != 1:
             raise ValueError("Removing Profile Image failed")
 
@@ -421,7 +431,7 @@ class Chat(object):
         """return True if this chat has location-sending enabled currently.
         :returns: True if location sending is enabled.
         """
-        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
+        return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
 
     def is_archived(self):
         """return True if this chat is archived.
@@ -434,7 +444,7 @@ class Chat(object):
 
         all subsequent messages will carry a location with them.
         """
-        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
+        lib.dc_send_locations_to_chat(self.account._dc_context, self.id, seconds)
 
     def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
         """return list of locations for the given contact in the given timespan.
@@ -458,7 +468,7 @@ class Chat(object):
         else:
             contact_id = contact.id
 
-        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
+        dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to)
         return [
             Location(
                 latitude=lib.dc_array_get_latitude(dc_array, i),

python/src/deltachat/const.py

@@ -1,197 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -3,6 +3,8 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
+from .chat import Chat
+from . import const
 
 
 class Contact(object):
@@ -11,23 +13,24 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
-        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
-        return self._dc_context == other._dc_context and self.id == other.id
+        return self.account._dc_context == other.account._dc_context and self.id == other.id
 
     def __ne__(self, other):
         return not (self == other)
 
     def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self.account._dc_context)
 
     @property
     def _dc_contact(self):
         return ffi.gc(
-            lib.dc_get_contact(self._dc_context, self.id),
+            lib.dc_get_contact(self.account._dc_context, self.id),
             lib.dc_contact_unref
         )
 
@@ -37,10 +40,13 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def display_name(self):
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
+    # deprecated alias
+    display_name = name
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -59,6 +65,16 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)
+    def create_chat(self):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        dc_context = self.account._dc_context
+        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
+        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
+        return Chat(self.account, chat_id)
+
+    # deprecated name
+    get_chat = create_chat

python/src/deltachat/direct_imap.py

@@ -0,0 +1,213 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
+import io
+import email
+import ssl
+import pathlib
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
+
+
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+ALL = "1:*"
+
+
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account):
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        ssl_context = ssl.create_default_context()
+
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
+        try:
+            self.conn.logout()
+        except (OSError, IMAPClientError):
+            print("Could not logout direct_imap conn")
+
+    def select_folder(self, foldername):
+        assert not self._idling
+        return self.conn.select_folder(foldername)
+
+    def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
+        if "_" not in config_name:
+            config_name = "configured_{}_folder".format(config_name)
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+
+    def list_folders(self):
+        """ return list of all existing folder names"""
+        assert not self._idling
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
+
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self):
+        assert not self._idling
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
+        assert not self._idling
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.set_flags(messages, [SEEN])
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self):
+        return len(self.get_unread_messages())
+
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[HEADER]']
+                flags = data[FLAGS]
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/eventlogger.py

@@ -1,137 +0,0 @@
-import deltachat
-import threading
-import time
-import re
-from queue import Queue, Empty
-from .hookspec import account_hookimpl, global_hookimpl
-
-
-@global_hookimpl
-def dc_account_init(account):
-    # send all FFI events for this account to a plugin hook
-    def _ll_event(ctx, evt_name, data1, data2):
-        assert ctx == account._dc_context
-        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-        account._pm.hook.ac_process_ffi_event(
-            account=account, ffi_event=ffi_event
-        )
-    deltachat.set_context_callback(account._dc_context, _ll_event)
-
-
-@global_hookimpl
-def dc_account_after_shutdown(dc_context):
-    deltachat.clear_context_callback(dc_context)
-
-
-class FFIEvent:
-    def __init__(self, name, data1, data2):
-        self.name = name
-        self.data1 = data1
-        self.data2 = data2
-
-    def __str__(self):
-        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
-
-
-class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
-    you'll get all ffi-events printed.
-    """
-    # to prevent garbled logging
-    _loglock = threading.RLock()
-
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
-        self.account = account
-        self.logid = logid
-        self.init_time = time.time()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._log_event(ffi_event)
-
-    def _log_event(self, ffi_event):
-        # don't show events that are anyway empty impls now
-        if ffi_event.name == "DC_EVENT_GET_STRING":
-            return
-        self.account.ac_log_line(str(ffi_event))
-
-    @account_hookimpl
-    def ac_log_line(self, message):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        elapsed = time.time() - self.init_time
-        locname = tname
-        if self.logid:
-            locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-        with self._loglock:
-            print(s, flush=True)
-
-
-class FFIEventTracker:
-    def __init__(self, account, timeout=None):
-        self.account = account
-        self._timeout = timeout
-        self._event_queue = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._event_queue.put(ffi_event)
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get(check_error=check_error)
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout if timeout is not None else self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError(str(ev))
-        return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev.name):
-                return ev
-
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
-                return ev
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.account.get_message_by_id(ev.data2)
-
-    def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
-        if the event contains no msgid"""
-        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-        if ev.data2 > 0:
-            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/events.py

@@ -0,0 +1,219 @@
+import threading
+import time
+import re
+from queue import Queue, Empty
+
+import deltachat
+from .hookspec import account_hookimpl
+from contextlib import contextmanager
+from .capi import ffi, lib
+from .message import map_system_message
+from .cutil import from_dc_charpointer
+
+
+class FFIEvent:
+    def __init__(self, name, data1, data2):
+        self.name = name
+        self.data1 = data1
+        self.data2 = data2
+
+    def __str__(self):
+        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
+
+
+class FFIEventLogger:
+    """ If you register an instance of this logger with an Account
+    you'll get all ffi-events printed.
+    """
+    # to prevent garbled logging
+    _loglock = threading.RLock()
+
+    def __init__(self, account):
+        self.account = account
+        self.logid = self.account.get_config("displayname")
+        self.init_time = time.time()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self.account.log(str(ffi_event))
+
+    @account_hookimpl
+    def ac_log_line(self, message):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        elapsed = time.time() - self.init_time
+        locname = tname
+        if self.logid:
+            locname += "-" + self.logid
+        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        with self._loglock:
+            print(s, flush=True)
+
+
+class FFIEventTracker:
+    def __init__(self, account, timeout=None):
+        self.account = account
+        self._timeout = timeout
+        self._event_queue = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._event_queue.put(ffi_event)
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get(check_error=check_error)
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout if timeout is not None else self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev.name == "DC_EVENT_ERROR":
+            raise ValueError("unexpected event: {}".format(ev))
+        return ev
+
+    def iter_events(self, timeout=None, check_error=True):
+        while 1:
+            yield self.get(timeout=timeout, check_error=check_error)
+
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        for ev in self.iter_events(timeout=timeout, check_error=check_error):
+            if rex.match(ev.name):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev.data2):
+                return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev.name), "event found {}".format(ev)
+
+    def wait_securejoin_inviter_progress(self, target):
+        while 1:
+            event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+            if event.data2 >= target:
+                print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
+                break
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.account.get_message_by_id(ev.data2)
+
+    def wait_next_messages_changed(self):
+        """ wait for and return next message-changed message or None
+        if the event contains no msgid"""
+        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
+        if ev.data2 > 0:
+            return self.account.get_message_by_id(ev.data2)
+
+    def wait_msg_delivered(self, msg):
+        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == msg.chat.id
+        assert ev.data2 == msg.id
+        assert msg.is_out_delivered()
+
+
+class EventThread(threading.Thread):
+    """ Event Thread for an account.
+
+    With each Account init this callback thread is started.
+    """
+    def __init__(self, account):
+        self.account = account
+        super(EventThread, self).__init__(name="events")
+        self.setDaemon(True)
+        self.start()
+
+    @contextmanager
+    def log_execution(self, message):
+        self.account.log(message + " START")
+        yield
+        self.account.log(message + " FINISHED")
+
+    def wait(self):
+        if self == threading.current_thread():
+            # we are in the callback thread and thus cannot
+            # wait for the thread-loop to finish.
+            return
+        self.join()
+
+    def run(self):
+        """ get and run events until shutdown. """
+        with self.log_execution("EVENT THREAD"):
+            self._inner_run()
+
+    def _inner_run(self):
+        event_emitter = ffi.gc(
+            lib.dc_get_event_emitter(self.account._dc_context),
+            lib.dc_event_emitter_unref,
+        )
+        while 1:
+            event = lib.dc_get_next_event(event_emitter)
+            if event == ffi.NULL:
+                break
+            evt = lib.dc_event_get_id(event)
+            data1 = lib.dc_event_get_data1_int(event)
+            # the following code relates to the deltachat/_build.py's helper
+            # function which provides us signature info of an event call
+            evt_name = deltachat.get_dc_event_name(evt)
+            if lib.dc_event_has_string_data(evt):
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
+            else:
+                data2 = lib.dc_event_get_data2_int(event)
+
+            lib.dc_event_unref(event)
+            ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+            try:
+                self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
+                for name, kwargs in self._map_ffi_event(ffi_event):
+                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
+                    hook = getattr(self.account._pm.hook, name)
+                    hook(**kwargs)
+            except Exception:
+                if self.account._dc_context is not None:
+                    raise
+
+    def _map_ffi_event(self, ffi_event):
+        name = ffi_event.name
+        account = self.account
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+            data1 = ffi_event.data1
+            if data1 == 0 or data1 == 1000:
+                success = data1 == 1000
+                yield "ac_configure_completed", dict(success=success)
+        elif name == "DC_EVENT_INCOMING_MSG":
+            msg = account.get_message_by_id(ffi_event.data2)
+            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+        elif name == "DC_EVENT_MSGS_CHANGED":
+            if ffi_event.data2 != 0:
+                msg = account.get_message_by_id(ffi_event.data2)
+                if msg.is_outgoing():
+                    res = map_system_message(msg)
+                    if res and res[0].startswith("ac_member"):
+                        yield res
+                    yield "ac_outgoing_message", dict(message=msg)
+                elif msg.is_in_fresh():
+                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+        elif name == "DC_EVENT_MSG_DELIVERED":
+            msg = account.get_message_by_id(ffi_event.data2)
+            yield "ac_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_CHAT_MODIFIED":
+            chat = account.get_chat_by_id(ffi_event.data1)
+            yield "ac_chat_modified", dict(chat=chat)

python/src/deltachat/hookspec.py

@@ -15,8 +15,9 @@ global_hookimpl = pluggy.HookimplMarker(global_spec_name)
 class PerAccount:
     """ per-Account-instance hook specifications.
 
-    Except for ac_process_ffi_event all hooks are executed
-    in the thread which calls Account.wait_shutdown().
+    All hooks are executed in a dedicated Event thread.
+    Hooks are not allowed to block/last long as this
+    blocks overall event processing on the python side.
     """
     @classmethod
     def _make_plugin_manager(cls):
@@ -42,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -88,5 +89,13 @@ class Global:
         """ called when `Account::__init__()` function starts executing. """
 
     @global_hookspec
-    def dc_account_after_shutdown(self, account, dc_context):
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
+    @global_hookspec
+    def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -1,106 +0,0 @@
-
-import threading
-import time
-
-from contextlib import contextmanager
-
-from .capi import lib
-
-
-class IOThreads:
-    def __init__(self, account):
-        self.account = account
-        self._dc_context = account._dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-
-    def is_started(self):
-        return len(self._name2thread) > 0
-
-    def start(self, callback_thread):
-        assert not self.is_started()
-        self._start_one_thread("inbox", self.imap_thread_run)
-        self._start_one_thread("smtp", self.smtp_thread_run)
-
-        if callback_thread:
-            self._start_one_thread("cb", self.cb_thread_run)
-
-        if int(self.account.get_config("mvbox_watch")):
-            self._start_one_thread("mvbox", self.mvbox_thread_run)
-
-        if int(self.account.get_config("sentbox_watch")):
-            self._start_one_thread("sentbox", self.sentbox_thread_run)
-
-    def _start_one_thread(self, name, func):
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        t.setDaemon(1)
-        t.start()
-
-    @contextmanager
-    def log_execution(self, message):
-        self.account.ac_log_line(message + " START")
-        yield
-        self.account.ac_log_line(message + " FINISHED")
-
-    def stop(self, wait=False):
-        self._thread_quitflag = True
-
-        # Workaround for a race condition. Make sure that thread is
-        # not in between checking for quitflag and entering idle.
-        time.sleep(0.5)
-
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        if "mvbox" in self._name2thread:
-            lib.dc_interrupt_mvbox_idle(self._dc_context)
-        if "sentbox" in self._name2thread:
-            lib.dc_interrupt_sentbox_idle(self._dc_context)
-        if wait:
-            for name, thread in self._name2thread.items():
-                if thread != threading.currentThread():
-                    thread.join()
-
-    def cb_thread_run(self):
-        with self.log_execution("CALLBACK THREAD START"):
-            it = self.account.iter_events()
-            while not self._thread_quitflag:
-                try:
-                    ev = next(it)
-                except StopIteration:
-                    break
-                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
-                ev.call_hook()
-
-    def imap_thread_run(self):
-        with self.log_execution("INBOX THREAD START"):
-            while not self._thread_quitflag:
-                lib.dc_perform_imap_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_idle(self._dc_context)
-
-    def mvbox_thread_run(self):
-        with self.log_execution("MVBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_mvbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_idle(self._dc_context)
-
-    def sentbox_thread_run(self):
-        with self.log_execution("SENTBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_sentbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_idle(self._dc_context)
-
-    def smtp_thread_run(self):
-        with self.log_execution("SMTP THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_smtp_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -16,8 +16,7 @@ class Message(object):
     """
     def __init__(self, account, dc_msg):
         self.account = account
-        self._dc_context = account._dc_context
-        assert isinstance(self._dc_context, ffi.CData)
+        assert isinstance(self.account._dc_context, ffi.CData)
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
@@ -29,8 +28,10 @@ class Message(object):
 
     def __repr__(self):
         c = self.get_sender_contact()
-        return "<Message id={} sender={}/{} outgoing={} chat={}/{}>".format(
-            self.id, c.id, c.addr, self.is_outgoing(), self.chat.id, self.chat.get_name())
+        typ = "outgoing" if self.is_outgoing() else "incoming"
+        return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
+            typ, self.is_system_message(), repr(self.text[:10]),
+            self.id, c.id, c.addr, self.chat.id, self.chat.get_name())
 
     @classmethod
     def from_db(cls, account, id):
@@ -52,15 +53,19 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
+    def create_chat(self):
+        """ create or get an existing chat (group) object for this message.
+
+        If the message is a deaddrop contact request
+        the sender will become an accepted contact.
+
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self._dc_context, self.id),
-                lib.dc_msg_unref
-        )
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):
@@ -95,7 +100,7 @@ class Message(object):
 
     def is_system_message(self):
         """ return True if this message is a system/info message. """
-        return lib.dc_msg_is_info(self._dc_msg)
+        return bool(lib.dc_msg_is_info(self._dc_msg))
 
     def is_setup_message(self):
         """ return True if this message is a setup message. """
@@ -118,12 +123,12 @@ class Message(object):
 
         The text is multiline and may contain eg. the raw text of the message.
         """
-        return from_dc_charpointer(lib.dc_get_msg_info(self._dc_context, self.id))
+        return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
 
     def continue_key_transfer(self, setup_code):
         """ extract key and use it as primary key for this account. """
         res = lib.dc_continue_key_transfer(
-                self._dc_context,
+                self.account._dc_context,
                 self.id,
                 as_dc_charpointer(setup_code)
         )
@@ -158,7 +163,7 @@ class Message(object):
         :returns: email-mime message object (with headers only, no body).
         """
         import email.parser
-        mime_headers = lib.dc_get_mime_headers(self._dc_context, self.id)
+        mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
             if isinstance(s, bytes):
@@ -201,7 +206,7 @@ class Message(object):
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
-                lib.dc_get_msg(self._dc_context, self.id),
+                lib.dc_get_msg(self.account._dc_context, self.id),
                 lib.dc_msg_unref
             )
         return lib.dc_msg_get_state(dc_msg)

python/src/deltachat/testplugin.py

@@ -1,21 +1,25 @@
 from __future__ import print_function
 import os
 import sys
+import io
 import subprocess
 import queue
 import threading
 import fnmatch
+import time
+import weakref
+import tempfile
+
 import pytest
 import requests
-import time
-from . import Account, const
-from .tracker import ConfigureTracker
-from .capi import lib
-from .eventlogger import FFIEventLogger, FFIEventTracker
-from _pytest.monkeypatch import MonkeyPatch
-from _pytest._code import Source
 
-import tempfile
+from . import Account, const
+from .capi import lib
+from .events import FFIEventLogger, FFIEventTracker
+from _pytest._code import Source
+from deltachat import direct_imap
+
+import deltachat
 
 
 def pytest_addoption(parser):
@@ -31,38 +35,80 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
         if cfg:
             config.option.liveconfig = cfg
 
+    # Make sure we don't get garbled output because threads keep running
+    # collect all ever created accounts in a weakref-set (so we don't
+    # keep objects unneccessarily alive) and enable/disable logging
+    # for each pytest test phase # (setup/call/teardown).
+    # Additionally make the acfactory use a logging/no-logging default.
 
-def pytest_runtest_setup(item):
-    if (list(item.iter_markers(name="ignored"))
-            and not item.config.getoption("ignored")):
-        pytest.skip("Ignored tests not requested, use --ignored")
+    class LoggingAspect:
+        def __init__(self):
+            self._accounts = weakref.WeakSet()
+
+        @deltachat.global_hookimpl
+        def dc_account_init(self, account):
+            self._accounts.add(account)
+
+        def disable_logging(self, item):
+            for acc in self._accounts:
+                acc.disable_logging()
+            acfactory = item.funcargs.get("acfactory")
+            if acfactory:
+                acfactory.set_logging_default(False)
+
+        def enable_logging(self, item):
+            for acc in self._accounts:
+                acc.enable_logging()
+            acfactory = item.funcargs.get("acfactory")
+            if acfactory:
+                acfactory.set_logging_default(True)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_runtest_setup(self, item):
+            if item.get_closest_marker("ignored"):
+                if not item.config.getvalue("ignored"):
+                    pytest.skip("use --ignored to run this test")
+            self.enable_logging(item)
+            yield
+            self.disable_logging(item)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_pyfunc_call(self, pyfuncitem):
+            self.enable_logging(pyfuncitem)
+            yield
+            self.disable_logging(pyfuncitem)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_runtest_teardown(self, item):
+            self.enable_logging(item)
+            yield
+            self.disable_logging(item)
+
+    la = LoggingAspect()
+    config.pluginmanager.register(la)
+    deltachat.register_global_plugin(la)
 
 
 def pytest_report_header(config, startdir):
     summary = []
 
     t = tempfile.mktemp()
-    m = MonkeyPatch()
     try:
-        m.setattr(sys.stdout, "write", lambda x: len(x))
         ac = Account(t)
         info = ac.get_info()
         ac.shutdown()
     finally:
-        m.undo()
         os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
+    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
          info['deltachat_core_version'],
          info['sqlite_version'],
+         info['journal_mode'],
      )])
 
     cfg = config.option.liveconfig
@@ -164,24 +210,37 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
+            self._accounts = []
             self.init_time = time.time()
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
+            self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
                 fin = self._finalizers.pop()
                 fin()
 
+            while self._accounts:
+                acc = self._accounts.pop()
+                acc.shutdown()
+                acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
+
         def make_account(self, path, logid, quiet=False):
-            ac = Account(path)
+            ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
+            ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
-            self._finalizers.append(ac.shutdown)
+                ac.add_account_plugin(FFIEventLogger(ac))
+            self._accounts.append(ac)
             return ac
 
+        def set_logging_default(self, logging):
+            self._logging = bool(logging)
+
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
@@ -244,24 +303,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac.start()
+            ac._configtracker = ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1._configtracker.wait_imap_connected()
-            ac1._configtracker.wait_smtp_connected()
-            ac1._configtracker.wait_finish()
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1._configtracker.wait_finish()
-            ac2._configtracker.wait_finish()
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+                        for i in range(num)]
+            self.wait_configure_and_start_io()
+            for acc in accounts:
+                acc.add_account_plugin(FFIEventLogger(acc))
+            return accounts
+
         def clone_online_account(self, account, pre_generated_key=True):
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
@@ -277,14 +341,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac.start()
+            ac._configtracker = ac.configure()
             return ac
 
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
                 "-u",
@@ -309,9 +388,42 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    return am
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
@@ -334,6 +446,7 @@ class BotProcess:
                     break
                 line = line.strip()
                 self.stdout_queue.put(line)
+                print("bot-stdout: ", line)
         finally:
             self.stdout_queue.put(None)
 
@@ -378,4 +491,17 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
+
     return Printer()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # set a report attribute for each phase of a call, which can
+    # be "setup", "call", "teardown"
+
+    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -18,7 +18,7 @@ class ImexTracker:
         if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
             self._imex_events.put(ffi_event.data1)
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(ffi_event.data1)
+            self._imex_events.put(ffi_event.data2)
 
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
@@ -40,11 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
+        self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -53,10 +56,15 @@ class ConfigureTracker:
             self._smtp_finished.set()
         elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
             self._imap_finished.set()
+        elif ffi_event.name == "DC_EVENT_CONFIGURE_PROGRESS":
+            self._progress.put(ffi_event.data1)
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
+        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """
@@ -66,6 +74,12 @@ class ConfigureTracker:
         """ wait until smtp is configured. """
         self._imap_finished.wait()
 
+    def wait_progress(self, data1=None):
+        while 1:
+            evdata = self._progress.get()
+            if data1 is None or evdata == data1:
+                break
+
     def wait_finish(self):
         """ wait until configure is completed.
 

python/tests/auditwheels.py

@@ -10,4 +10,6 @@ if __name__ == "__main__":
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
-            subprocess.check_call(["auditwheel", "repair", p, "-w", workspacedir])
+            subprocess.check_call(
+                ["auditwheel", "repair", p, "-w", workspacedir,
+                 "--plat", "manylinux2014_x86_64"])

python/tests/conftest.py

@@ -1,18 +0,0 @@
-from __future__ import print_function
-
-
-def wait_configuration_progress(account, min_target, max_target=1001):
-    min_target = min(min_target, max_target)
-    while 1:
-        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        if event.data1 >= min_target and event.data1 <= max_target:
-            print("** CONFIG PROGRESS {}".format(min_target), account)
-            break
-
-
-def wait_securejoin_inviter_progress(account, target):
-    while 1:
-        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-        if event.data2 >= target:
-            print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
-            break

python/tests/stress_test_db.py

@@ -0,0 +1,135 @@
+
+import time
+import threading
+import pytest
+import os
+from queue import Queue, Empty
+
+import deltachat
+
+
+def test_db_busy_error(acfactory, tmpdir):
+    starttime = time.time()
+    log_lock = threading.RLock()
+
+    def log(string):
+        with log_lock:
+            print("%3.2f %s" % (time.time() - starttime, string))
+
+    # make a number of accounts
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
+    log("created %s accounts" % len(accounts))
+
+    # put a bigfile into each account
+    for acc in accounts:
+        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
+        with open(acc.bigfile, "wb") as f:
+            f.write(b"01234567890"*1000_000)
+    log("created %s bigfiles" % len(accounts))
+
+    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
+    chat = accounts[0].create_group_chat("stress-group")
+    for addr in contact_addrs[1:]:
+        chat.add_contact(chat.account.create_contact(addr))
+
+    # setup auto-responder bots which report back failures/actions
+    report_queue = Queue()
+
+    def report_func(replier, report_type, *report_args):
+        report_queue.put((replier, report_type, report_args))
+
+    # each replier receives all events and sends report events to receive_queue
+    repliers = []
+    for acc in accounts:
+        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
+        acc.add_account_plugin(replier)
+        repliers.append(replier)
+
+    # kick off message sending
+    # after which repliers will reply to each other
+    chat.send_text("hello")
+
+    alive_count = len(accounts)
+    while alive_count > 0:
+        try:
+            replier, report_type, report_args = report_queue.get(timeout=10)
+        except Empty:
+            log("timeout waiting for next event")
+            pytest.fail("timeout exceeded")
+        if report_type == ReportType.exit:
+            replier.log("EXIT")
+        elif report_type == ReportType.ffi_error:
+            replier.log("ERROR: {}".format(report_args[0]))
+        elif report_type == ReportType.message_echo:
+            continue
+        else:
+            raise ValueError("{} unknown report type {}, args={}".format(
+                addr, report_type, report_args
+            ))
+        alive_count -= 1
+        replier.log("shutting down")
+        replier.account.shutdown()
+        replier.log("shut down complete, remaining={}".format(alive_count))
+
+
+class ReportType:
+    exit = "exit"
+    ffi_error = "ffi-error"
+    message_echo = "message-echo"
+
+
+class AutoReplier:
+    def __init__(self, account, log, num_send, num_bigfiles, report_func):
+        self.account = account
+        self._log = log
+        self.report_func = report_func
+        self.num_send = num_send
+        self.num_bigfiles = num_bigfiles
+        self.current_sent = 0
+        self.addr = self.account.get_self_contact().addr
+
+        self._thread = threading.Thread(
+            name="Stats{}".format(self.account),
+            target=self.thread_stats
+        )
+        self._thread.setDaemon(True)
+        self._thread.start()
+
+    def log(self, message):
+        self._log("{} {}".format(self.addr, message))
+
+    def thread_stats(self):
+        # XXX later use, for now we just quit
+        return
+        while 1:
+            time.sleep(1.0)
+            break
+
+    @deltachat.account_hookimpl
+    def ac_incoming_message(self, message):
+        if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
+            return
+        message.create_chat()
+        message.mark_seen()
+        self.log("incoming message: {}".format(message))
+
+        self.current_sent += 1
+        # we are still alive, let's send a reply
+        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
+            message.chat.send_text("send big file as reply to: {}".format(message.text))
+            msg = message.chat.send_file(self.account.bigfile)
+        else:
+            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
+            assert msg.text
+        self.log("message-sent: {}".format(msg))
+        self.report_func(self, ReportType.message_echo)
+        if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
+            return
+
+    @deltachat.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self.log(ffi_event)
+        if ffi_event.name == "DC_EVENT_ERROR":
+            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_increation.py

@@ -6,27 +6,37 @@ import shutil
 import pytest
 from filecmp import cmp
 
-from conftest import wait_configuration_progress
 from deltachat import const
 
 
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev.data1 == chat_id
-    if msg_id is not None:
-        assert ev.data2 == msg_id
-    return ev.data2
+def wait_msg_delivered(account, msg_list):
+    """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
+    msg_list = list(msg_list)
+    while msg_list:
+        ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        msg_list.remove((ev.data1, ev.data2))
+
+
+def wait_msgs_changed(account, msgs_list):
+    """ wait for one or more MSGS_CHANGED events to match msgs_list contents. """
+    account.log("waiting for msgs_list={}".format(msgs_list))
+    msgs_list = list(msgs_list)
+    while msgs_list:
+        ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        for i, (data1, data2) in enumerate(msgs_list):
+            if ev.data1 == data1:
+                if data2 is None or ev.data2 == data2:
+                    del msgs_list[i]
+                    break
+        else:
+            account.log("waiting mismatch data1={} data2={}".format(data1, data2))
+    return ev.data1, ev.data2
 
 
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -35,13 +45,8 @@ class TestOnlineInCreation:
             chat.prepare_message_file(src.strpath)
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -53,15 +58,10 @@ class TestOnlineInCreation:
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac1, 1000)
-        wait_configuration_progress(ac2, 1000)
+        ac1, ac2 = acfactory.get_two_online_accounts()
 
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
-        wait_msgs_changed(ac1, 0, 0)  # why no chat id?
+        chat = ac1.create_chat(ac2)
+        wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
         orig = data.get_path("d.png")
@@ -70,19 +70,16 @@ class TestOnlineInCreation:
             fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
-        wait_msgs_changed(ac1, chat.id, prepared_original.id)
+        wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(c2)
-        wait_msgs_changed(ac1, 0, 0)  # why not chat id?
+        chat2.add_contact(ac2)
+        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them
         # is the one caused by forwarding
-        forwarded_id = wait_msgs_changed(ac1, chat2.id)
-        if forwarded_id == 0:
-            forwarded_id = wait_msgs_changed(ac1, chat2.id)
-            assert forwarded_id
+        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 
@@ -91,20 +88,18 @@ class TestOnlineInCreation:
         shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
-        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
-        lp.sec("expect the forwarded message to be sent now too")
-        wait_msgs_changed(ac1, chat2.id, forwarded_id)
+        lp.sec("check that both forwarded and original message are proper.")
+        wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
+
         fwd_msg = ac1.get_message_by_id(forwarded_id)
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
-        lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat.id
-        assert ev.data2 == prepared_original.id
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat2.id
-        assert ev.data2 == forwarded_id
+        lp.sec("wait for both messages to be delivered to SMTP")
+        wait_msg_delivered(ac1, [
+            (chat2.id, forwarded_id),
+            (chat.id, prepared_original.id)
+        ])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
         ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")

python/tests/test_lowlevel.py

@@ -1,75 +1,52 @@
 from __future__ import print_function
-from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
+
+from queue import Queue
+from deltachat import capi, cutil, const
 from deltachat import register_global_plugin
 from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
+# from deltachat.account import EventLogger
 
 
 def test_empty_context():
     ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
-    capi.lib.dc_close(ctx)
-
-
-def test_callback_None2int():
-    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
-    set_context_callback(ctx, lambda *args: None)
-    capi.lib.dc_close(ctx)
-    clear_context_callback(ctx)
+    capi.lib.dc_context_unref(ctx)
 
 
 def test_dc_close_events(tmpdir, acfactory):
     ac1 = acfactory.get_unconfigured_account()
 
     # register after_shutdown function
-    shutdowns = []
+    shutdowns = Queue()
 
     class ShutdownPlugin:
         @global_hookimpl
         def dc_account_after_shutdown(self, account):
             assert account._dc_context is None
-            shutdowns.append(account)
+            shutdowns.put(account)
     register_global_plugin(ShutdownPlugin())
     assert hasattr(ac1, "_dc_context")
     ac1.shutdown()
-    assert shutdowns == [ac1]
-
-    def find(info_string):
-        evlog = ac1._evtracker
-        while 1:
-            ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
-            data2 = ev.data2
-            if info_string in data2:
-                return
-            else:
-                print("skipping event", ev)
-
-    find("disconnecting inbox-thread")
-    find("disconnecting sentbox-thread")
-    find("disconnecting mvbox-thread")
-    find("disconnecting SMTP")
-    find("Database closed")
+    shutdowns.get(timeout=2)
 
 
 def test_wrong_db(tmpdir):
-    dc_context = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
     p = tmpdir.join("hello.db")
     # write an invalid database file
     p.write("x123" * 10)
-    assert not lib.dc_open(dc_context, p.strpath.encode("ascii"), ffi.NULL)
+
+    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
 
 
 def test_empty_blobdir(tmpdir):
+    db_fname = tmpdir.join("hello.db")
     # Apparently some client code expects this to be the same as passing NULL.
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
         lib.dc_context_unref,
     )
-    db_fname = tmpdir.join("hello.db")
-    assert lib.dc_open(ctx, db_fname.strpath.encode("ascii"), b"")
+    assert ctx != ffi.NULL
 
 
 def test_event_defines():
@@ -78,19 +55,22 @@ def test_event_defines():
 
 
 def test_sig():
-    sig = capi.lib.dc_get_event_signature_types
-    assert sig(const.DC_EVENT_INFO) == 2
-    assert sig(const.DC_EVENT_WARNING) == 2
-    assert sig(const.DC_EVENT_ERROR) == 2
-    assert sig(const.DC_EVENT_SMTP_CONNECTED) == 2
-    assert sig(const.DC_EVENT_IMAP_CONNECTED) == 2
-    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT) == 2
+    sig = capi.lib.dc_event_has_string_data
+    assert not sig(const.DC_EVENT_MSGS_CHANGED)
+    assert sig(const.DC_EVENT_INFO)
+    assert sig(const.DC_EVENT_WARNING)
+    assert sig(const.DC_EVENT_ERROR)
+    assert sig(const.DC_EVENT_SMTP_CONNECTED)
+    assert sig(const.DC_EVENT_IMAP_CONNECTED)
+    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
+    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
 
 
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
-    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    chat = ac1.create_chat_by_contact(contact1)
+
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
@@ -107,47 +87,18 @@ def test_get_special_message_id_returns_empty_message(acfactory):
 
 def test_provider_info_none():
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
 
 
-def test_get_info_closed():
-    ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
-    assert 'deltachat_core_version' in info
-    assert 'database_dir' not in info
-
-
 def test_get_info_open(tmpdir):
+    db_fname = tmpdir.join("test.db")
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
         lib.dc_context_unref,
     )
-    db_fname = tmpdir.join("test.db")
-    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
     assert 'deltachat_core_version' in info
     assert 'database_dir' in info
-
-
-def test_is_open_closed():
-    ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    assert lib.dc_is_open(ctx) == 0
-
-
-def test_is_open_actually_open(tmpdir):
-    ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    db_fname = tmpdir.join("test.db")
-    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
-    assert lib.dc_is_open(ctx) == 1

python/tox.ini

@@ -71,6 +71,8 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
+markers = 
+    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

rust-toolchain

@@ -1 +1 @@
-nightly-2020-03-12
+1.43.1

spec.md

@@ -1,10 +1,12 @@
-# Chat-over-Email specification
+# chat-mail specification
 
-Version 0.30.0
+Version: 0.32.0
+Status:  In-progress 
+Format:  [Semantic Line Breaks](https://sembr.org/)
 
-This document describes how emails can be used
-to implement typical messenger functions
-while staying compatible to existing MUAs.
+This document roughly describes how chat-mail 
+apps use the standard e-mail system 
+to implement typical messenger functions.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -30,17 +32,14 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
-If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD start with the characters `Chat:`
-and SHOULD be an excerpt of the message.
+the `Subject` header SHOULD be `Message from <sender name>`.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -58,7 +57,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Chat: Hello ...
+    Subject: Message from sender@domain
 
     Hello world!
 

src/aheader.rs

@@ -6,13 +6,15 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
+use deltachat_derive::*;
+
 use crate::contact::*;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};
 
 /// Possible values for encryption preference
-#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive)]
+#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive, Sqlx)]
 #[repr(u8)]
 pub enum EncryptPreference {
     NoPreference = 0,

src/blob.rs

@@ -2,16 +2,21 @@
 
 use std::ffi::OsStr;
 use std::fmt;
-use std::fs;
-use std::io::Write;
-use std::path::{Path, PathBuf};
+
+use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
+use async_std::{fs, io};
 
 use image::GenericImageView;
+use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::constants::AVATAR_SIZE;
+use crate::config::Config;
+use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::Event;
+use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -43,30 +48,35 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WriteFailure] is used when the file could not
     /// be written to.  You can expect [BlobError.cause] to contain an
     /// underlying error.
-    pub fn create(
+    pub async fn create(
         context: &'a Context,
         suggested_name: impl AsRef<str>,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
-        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext)?;
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
         file.write_all(data)
+            .await
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
     // Creates a new file, returning a tuple of the name and the handle.
-    fn create_new_file(dir: &Path, stem: &str, ext: &str) -> Result<(String, fs::File), BlobError> {
+    async fn create_new_file(
+        dir: &Path,
+        stem: &str,
+        ext: &str,
+    ) -> Result<(String, fs::File), BlobError> {
         let max_attempt = 15;
         let mut name = format!("{}{}", stem, ext);
         for attempt in 0..max_attempt {
@@ -75,6 +85,7 @@ impl<'a> BlobObject<'a> {
                 .create_new(true)
                 .write(true)
                 .open(&path)
+                .await
             {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
@@ -110,37 +121,41 @@ impl<'a> BlobObject<'a> {
     /// In addition to the errors in [BlobObject::create] the
     /// [BlobError::CopyFailure] is used when the data can not be
     /// copied.
-    pub fn create_and_copy(
+    pub async fn create_and_copy(
         context: &'a Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file = fs::File::open(src.as_ref()).map_err(|err| BlobError::CopyFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: String::from(""),
-            src: src.as_ref().to_path_buf(),
-            cause: err,
-        })?;
+        let mut src_file =
+            fs::File::open(src.as_ref())
+                .await
+                .map_err(|err| BlobError::CopyFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: String::from(""),
+                    src: src.as_ref().to_path_buf(),
+                    cause: err,
+                })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
-        let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
+        let (name, mut dst_file) =
+            BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
-        std::io::copy(&mut src_file, &mut dst_file).map_err(|err| {
+        if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
             {
                 // Attempt to remove the failed file, swallow errors resulting from that.
                 let path = context.get_blobdir().join(&name_for_err);
-                fs::remove_file(path).ok();
+                fs::remove_file(path).await.ok();
             }
-            BlobError::CopyFailure {
+            return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
-            }
-        })?;
+            });
+        }
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -158,14 +173,14 @@ impl<'a> BlobObject<'a> {
     /// This merely delegates to the [BlobObject::create_and_copy] and
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
-    pub fn new_from_path(
+    pub async fn new_from_path(
         context: &Context,
         src: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject, BlobError> {
+    ) -> std::result::Result<BlobObject<'_>, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
         } else {
-            BlobObject::create_and_copy(context, src)
+            BlobObject::create_and_copy(context, src).await
         }
     }
 
@@ -361,11 +376,73 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
     }
+
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
+            != Some((Viewtype::Image, "image/jpeg"))
+        {
+            return Ok(());
+        }
+
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
+
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
+        }
+
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
+    }
+
+    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exifreader = exif::Reader::new();
+        let exif = exifreader.read_from_container(&mut bufreader)?;
+        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
+            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
+            // we only use rotation, in practise, flipping is not used.
+            match orientation.value.get_uint(0) {
+                Some(3) => return Ok(180),
+                Some(6) => return Ok(90),
+                Some(8) => return Ok(270),
+                other => warn!(context, "exif orientation value ignored: {:?}", other),
+            }
+        }
+        Ok(0)
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -389,7 +466,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
@@ -418,58 +495,71 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[test]
-    fn test_create() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_create() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
         let fname = t.ctx.get_blobdir().join("foo");
-        let data = fs::read(fname).unwrap();
+        let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
         assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
     }
 
-    #[test]
-    fn test_lowercase_ext() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_lowercase_ext() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
-    #[test]
-    fn test_as_file_name() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_as_file_name() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
-    #[test]
-    fn test_as_rel_path() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_as_rel_path() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
-    #[test]
-    fn test_suffix() {
-        let t = dummy_context();
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_suffix() {
+        let t = TestContext::new().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
+        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
-    #[test]
-    fn test_create_dup() {
-        let t = dummy_context();
-        BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_create_dup() {
+        let t = TestContext::new().await;
+        BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         let foo_path = t.ctx.get_blobdir().join("foo.txt");
-        assert!(foo_path.exists());
-        BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
-        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+        assert!(foo_path.exists().await);
+        BlobObject::create(&t.ctx, "foo.txt", b"world")
+            .await
+            .unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
-                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
+                assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 assert!(name.starts_with("foo"));
@@ -478,17 +568,22 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_double_ext_preserved() {
-        let t = dummy_context();
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello").unwrap();
+    #[async_std::test]
+    async fn test_double_ext_preserved() {
+        let t = TestContext::new().await;
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
+            .await
+            .unwrap();
         let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
-        assert!(foo_path.exists());
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"world").unwrap();
-        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+        assert!(foo_path.exists().await);
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
+            .await
+            .unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
-                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
+                assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 println!("{}", name);
@@ -498,55 +593,55 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_create_long_names() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_long_names() {
+        let t = TestContext::new().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t.ctx, &s, b"data").unwrap();
+        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
 
-    #[test]
-    fn test_create_and_copy() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_and_copy() {
+        let t = TestContext::new().await;
         let src = t.dir.path().join("src");
-        fs::write(&src, b"boo").unwrap();
-        let blob = BlobObject::create_and_copy(&t.ctx, &src).unwrap();
+        fs::write(&src, b"boo").await.unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
-        let data = fs::read(blob.to_abs_path()).unwrap();
+        let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).is_err());
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
         let whoops = t.ctx.get_blobdir().join("whoops");
-        assert!(!whoops.exists());
+        assert!(!whoops.exists().await);
     }
 
-    #[test]
-    fn test_create_from_path() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_from_path() {
+        let t = TestContext::new().await;
 
         let src_ext = t.dir.path().join("external");
-        fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        fs::write(&src_ext, b"boo").await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
-        let data = fs::read(blob.to_abs_path()).unwrap();
+        let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let src_int = t.ctx.get_blobdir().join("internal");
-        fs::write(&src_int, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).unwrap();
+        fs::write(&src_int, b"boo").await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
-        let data = fs::read(blob.to_abs_path()).unwrap();
+        let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
     }
-    #[test]
-    fn test_create_from_name_long() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_from_name_long() {
+        let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
-        fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        fs::write(&src_ext, b"boo").await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -565,8 +660,40 @@ mod tests {
 
     #[test]
     fn test_sanitise_name() {
-        let (_, ext) =
+        let (stem, ext) =
             BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
         assert_eq!(ext, ".txt");
+        assert!(!stem.is_empty());
+
+        // the extensions are kept together as between stem and extension a number may be added -
+        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
+        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
+        assert_eq!(stem, "wot");
+        assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
+        assert_eq!(stem, "");
+        assert_eq!(ext, ".foo.bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
+        assert!(stem.contains("foo"));
+        assert!(!stem.contains("?"));
+        assert_eq!(ext, ".bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("no-extension");
+        assert_eq!(stem, "no-extension");
+        assert_eq!(ext, "");
+
+        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
+        assert_eq!(ext, ".c");
+        assert!(!stem.contains("path"));
+        assert!(!stem.contains("ignored"));
+        assert!(stem.contains("this"));
+        assert!(stem.contains("forbidden"));
+        assert!(!stem.contains("/"));
+        assert!(!stem.contains("\\"));
+        assert!(!stem.contains(":"));
+        assert!(!stem.contains("*"));
+        assert!(!stem.contains("?"));
     }
 }

src/chatlist.rs

@@ -86,33 +86,28 @@ impl Chatlist {
     ///     are returned.
     /// `query_contact_id`: An optional contact ID for filtering the list. Only chats including this contact ID
     ///     are returned.
-    pub fn try_load(
+    pub async fn try_load(
         context: &Context,
         listflags: usize,
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
+        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
+        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
+        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
+        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
+
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context) {
+        if let Err(err) = delete_device_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
         let mut add_archived_link_item = false;
 
-        let process_row = |row: &rusqlite::Row| {
-            let chat_id: ChatId = row.get(0)?;
-            let msg_id: MsgId = row.get(1).unwrap_or_default();
-            Ok((chat_id, msg_id))
-        };
-
-        let process_rows = |rows: rusqlite::MappedRows<_>| {
-            rows.collect::<std::result::Result<Vec<_>, _>>()
-                .map_err(Into::into)
-        };
-
-        let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+        let skip_id = if flag_for_forwarding {
             chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+                .await
                 .unwrap_or_default()
                 .0
         } else {
@@ -136,110 +131,115 @@ impl Chatlist {
         // shown at all permanent in the chatlist.
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
-            context.sql.query_map(
+            context.sql.query_rows(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
-                process_row,
-                process_rows,
-            )?
-        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
+                 ORDER BY c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsx![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+            ).await?
+        } else if flag_archived_only {
             // show archived chats
             // (this includes the archived device-chat; we could skip it,
             // however, then the number of archived chats do not match, which might be even more irritating.
             // and adapting the number requires larger refactorings and seems not to be worth the effort)
-            context.sql.query_map(
-                "SELECT c.id, m.id
+            context
+                .sql
+                .query_rows(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft],
-                process_row,
-                process_rows,
-            )?
+                    paramsx![MessageState::OutDraft],
+                )
+                .await?
         } else if let Some(query) = query {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
 
             // allow searching over special names that may change at any time
             // when the ui calls set_stock_translation()
-            if let Err(err) = update_special_chat_names(context) {
+            if let Err(err) = update_special_chat_names(context).await {
                 warn!(context, "cannot update special chat names: {:?}", err)
             }
 
             let str_like_cmd = format!("%{}%", query);
-            context.sql.query_map(
-                "SELECT c.id, m.id
+            context
+                .sql
+                .query_rows(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND c.name LIKE ?3
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, str_like_cmd],
-                process_row,
-                process_rows,
-            )?
+                    paramsx![MessageState::OutDraft, skip_id, str_like_cmd],
+                )
+                .await?
         } else {
             //  show normal chatlist
-            let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+            let sort_id_up = if flag_for_forwarding {
                 chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+                    .await
                     .unwrap_or_default()
                     .0
             } else {
                 ChatId::new(0)
             };
-            let mut ids = context.sql.query_map(
+            let mut ids = context.sql.query_rows(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND NOT c.archived=?3
                  GROUP BY c.id
                  ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
-                process_row,
-                process_rows,
-            )?;
-            if 0 == listflags & DC_GCL_NO_SPECIALS {
-                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    if 0 == listflags & DC_GCL_FOR_FORWARDING {
+                paramsx![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+            ).await?;
+
+            if !flag_no_specials {
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context).await
+                {
+                    if !flag_for_forwarding {
                         ids.insert(
                             0,
                             (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
@@ -251,8 +251,8 @@ impl Chatlist {
             ids
         };
 
-        if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
-            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
+        if add_archived_link_item && dc_get_archived_cnt(context).await > 0 {
+            if ids.is_empty() && flag_add_alldone_hint {
                 ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
             ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
@@ -305,7 +305,7 @@ impl Chatlist {
     /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
     /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
     //    0 if not applicable.
-    pub fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
+    pub async fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
@@ -323,7 +323,7 @@ impl Chatlist {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) {
+        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
@@ -332,11 +332,11 @@ impl Chatlist {
 
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
-                lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
             }
 
             Some(lastmsg)
@@ -348,9 +348,15 @@ impl Chatlist {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(context.stock_str(StockMessage::NoMessages).to_string());
+            ret.text2 = Some(
+                context
+                    .stock_str(StockMessage::NoMessages)
+                    .await
+                    .to_string(),
+            );
         } else {
-            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context);
+            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
+                .await;
         }
 
         ret
@@ -362,34 +368,38 @@ impl Chatlist {
 }
 
 /// Returns the number of archived chats
-pub fn dc_get_archived_cnt(context: &Context) -> u32 {
-    context
+pub async fn dc_get_archived_cnt(context: &Context) -> u32 {
+    let v: i32 = context
         .sql
-        .query_get_value(
-            context,
+        .query_value(
             "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
-            params![],
+            paramsx![],
         )
-        .unwrap_or_default()
+        .await
+        .unwrap_or_default();
+    v as u32
 }
 
-fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
     // We have an index over the state-column, this should be
     // sufficient as there are typically only few fresh messages.
-    context.sql.query_get_value(
-        context,
-        concat!(
-            "SELECT m.id",
-            " FROM msgs m",
-            " LEFT JOIN chats c",
-            "        ON c.id=m.chat_id",
-            " WHERE m.state=10",
-            "   AND m.hidden=0",
-            "   AND c.blocked=2",
-            " ORDER BY m.timestamp DESC, m.id DESC;"
-        ),
-        params![],
-    )
+    context
+        .sql
+        .query_value(
+            r#"
+SELECT m.id
+  FROM msgs m
+  LEFT JOIN chats c
+        ON c.id=m.chat_id
+ WHERE m.state=10
+   AND m.hidden=0
+   AND c.blocked=2
+ ORDER BY m.timestamp DESC, m.id DESC;
+"#,
+            paramsx![],
+        )
+        .await
+        .ok()
 }
 
 #[cfg(test)]
@@ -398,15 +408,21 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[test]
-    fn test_try_load() {
-        let t = dummy_context();
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat").unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat").unwrap();
+    #[async_std::test]
+    async fn test_try_load() {
+        let t = TestContext::new().await;
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+            .await
+            .unwrap();
+        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
+            .await
+            .unwrap();
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
+            .await
+            .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -415,77 +431,102 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg));
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
             .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .await
             .ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 1);
     }
 
-    #[test]
-    fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context();
-        t.ctx.update_device_chats().unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+    #[async_std::test]
+    async fn test_sort_self_talk_up_on_forward() {
+        let t = TestContext::new().await;
+        t.ctx.update_device_chats().await.unwrap();
+        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+            .await
+            .unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
         assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
+            .await
+            .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
         assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .await
             .unwrap()
             .is_self_talk());
     }
 
-    #[test]
-    fn test_search_special_chat_names() {
-        let t = dummy_context();
-        t.ctx.update_device_chats().unwrap();
+    #[async_std::test]
+    async fn test_search_special_chat_names() {
+        let t = TestContext::new().await;
+        t.ctx.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 0);
 
         t.ctx
             .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+            .await
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+            .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
         assert_eq!(chats.len(), 1);
 
         t.ctx
             .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+            .await
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+            .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
         assert_eq!(chats.len(), 1);
     }
 
-    #[test]
-    fn test_get_summary_unwrap() {
-        let t = dummy_context();
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+    #[async_std::test]
+    async fn test_get_summary_unwrap() {
+        let t = TestContext::new().await;
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+            .await
+            .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg));
+        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None);
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t.ctx, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/config.rs

@@ -9,11 +9,9 @@ use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::events::Event;
-use crate::job::*;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
-use rusqlite::NO_PARAMS;
 
 /// The available configuration keys.
 #[derive(
@@ -65,6 +63,9 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
+    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
+    MediaQuality,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -103,6 +104,9 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
+    ConfiguredInboxFolder,
+    ConfiguredMvboxFolder,
+    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -116,17 +120,21 @@ pub enum Config {
 }
 
 impl Context {
+    pub async fn config_exists(&self, key: Config) -> bool {
+        self.sql.get_raw_config(key).await.is_some()
+    }
+
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
-    pub fn get_config(&self, key: Config) -> Option<String> {
+    pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_raw_config(self, key);
+                let rel_path = self.sql.get_raw_config(key).await;
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_raw_config(self, key),
+            _ => self.sql.get_raw_config(key).await,
         };
 
         if value.is_some() {
@@ -135,27 +143,29 @@ impl Context {
 
         // Default values
         match key {
-            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).into_owned()),
+            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
 
-    pub fn get_config_int(&self, key: Config) -> i32 {
+    pub async fn get_config_int(&self, key: Config) -> i32 {
         self.get_config(key)
+            .await
             .and_then(|s| s.parse().ok())
             .unwrap_or_default()
     }
 
-    pub fn get_config_bool(&self, key: Config) -> bool {
-        self.get_config_int(key) != 0
+    pub async fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key).await != 0
     }
 
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
     /// at once, `Some(x)` means delete after `x` seconds.
-    pub fn get_config_delete_server_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteServerAfter) {
+    pub async fn get_config_delete_server_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteServerAfter).await {
             0 => None,
             1 => Some(0),
             x => Some(x as i64),
@@ -166,8 +176,8 @@ impl Context {
     ///
     /// `None` means never delete the message, `Some(x)` means delete
     /// after `x` seconds.
-    pub fn get_config_delete_device_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteDeviceAfter) {
+    pub async fn get_config_delete_device_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteDeviceAfter).await {
             0 => None,
             x => Some(x as i64),
         }
@@ -175,57 +185,46 @@ impl Context {
 
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
+    pub async fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
             Config::Selfavatar => {
                 self.sql
-                    .execute("UPDATE contacts SET selfavatar_sent=0;", NO_PARAMS)?;
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", paramsx![])
+                    .await?;
                 self.sql
-                    .set_raw_config_bool(self, "attach_selfavatar", true)?;
+                    .set_raw_config_bool(self, "attach_selfavatar", true)
+                    .await?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(&self, value)?;
+                        let blob = BlobObject::new_from_path(&self, value).await?;
                         blob.recode_to_avatar_size(self)?;
-                        self.sql.set_raw_config(self, key, Some(blob.as_name()))
+                        self.sql
+                            .set_raw_config(self, key, Some(blob.as_name()))
+                            .await
                     }
-                    None => self.sql.set_raw_config(self, key, None),
+                    None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
-            Config::InboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_inbox_idle(self);
-                ret
-            }
-            Config::SentboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_sentbox_idle(self);
-                ret
-            }
-            Config::MvboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_mvbox_idle(self);
-                ret
-            }
             Config::Selfstatus => {
-                let def = self.stock_str(StockMessage::StatusLine);
+                let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
                     value
                 };
 
-                self.sql.set_raw_config(self, key, val)
+                self.sql.set_raw_config(self, key, val).await
             }
             Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
-                self.call_cb(Event::MsgsChanged {
+                self.emit_event(Event::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 });
                 ret
             }
-            _ => self.sql.set_raw_config(self, key, value),
+            _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
 }
@@ -248,9 +247,11 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::constants;
     use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
     use image::GenericImageView;
+    use num_traits::FromPrimitive;
     use std::fs::File;
     use std::io::Write;
 
@@ -271,9 +272,9 @@ mod tests {
         assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
     }
 
-    #[test]
-    fn test_selfavatar_outside_blobdir() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_selfavatar_outside_blobdir() {
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         File::create(&avatar_src)
@@ -281,13 +282,14 @@ mod tests {
             .write_all(avatar_bytes)
             .unwrap();
         let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists());
+        assert!(!avatar_blob.exists().await);
         t.ctx
             .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
             .unwrap();
-        assert!(avatar_blob.exists());
+        assert!(avatar_blob.exists().await);
         assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
@@ -299,9 +301,9 @@ mod tests {
         assert_eq!(img.height(), AVATAR_SIZE);
     }
 
-    #[test]
-    fn test_selfavatar_in_blobdir() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_selfavatar_in_blobdir() {
+        let t = TestContext::new().await;
         let avatar_src = t.ctx.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
@@ -315,8 +317,9 @@ mod tests {
 
         t.ctx
             .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
             .unwrap();
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
@@ -324,9 +327,9 @@ mod tests {
         assert_eq!(img.height(), AVATAR_SIZE);
     }
 
-    #[test]
-    fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
         File::create(&avatar_src)
@@ -334,16 +337,37 @@ mod tests {
             .write_all(avatar_bytes)
             .unwrap();
         let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists());
+        assert!(!avatar_blob.exists().await);
         t.ctx
             .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
             .unwrap();
-        assert!(avatar_blob.exists());
+        assert!(avatar_blob.exists().await);
         assert_eq!(
             std::fs::metadata(&avatar_blob).unwrap().len(),
             avatar_bytes.len() as u64
         );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
+
+    #[async_std::test]
+    async fn test_media_quality_config_option() {
+        let t = TestContext::new().await;
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        assert_eq!(media_quality, 0);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Balanced);
+
+        t.ctx
+            .set_config(Config::MediaQuality, Some("1"))
+            .await
+            .unwrap();
+
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        assert_eq!(media_quality, 1);
+        assert_eq!(constants::MediaQuality::Worse as i32, 1);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Worse);
+    }
 }

src/configure/auto_mozilla.rs

@@ -94,12 +94,12 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
     }
 }
 
-pub fn moz_autoconfigure(
+pub async fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
 ) -> Result<LoginParam, Error> {
-    let xml_raw = read_url(context, url)?;
+    let xml_raw = read_url(context, url).await?;
 
     let res = parse_xml(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/auto_outlook.rs

@@ -112,7 +112,7 @@ fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     Ok(res)
 }
 
-pub fn outlk_autodiscover(
+pub async fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
@@ -120,7 +120,7 @@ pub fn outlk_autodiscover(
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
-        let xml_raw = read_url(context, &url)?;
+        let xml_raw = read_url(context, &url).await?;
         let res = parse_xml(&xml_raw);
         if let Err(err) = &res {
             warn!(context, "{}", err);

src/configure/read_url.rs

@@ -3,17 +3,13 @@ use crate::context::Context;
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("URL request error")]
-    GetError(#[from] reqwest::Error),
+    GetError(surf::Error),
 }
 
-pub fn read_url(context: &Context, url: &str) -> Result<String, Error> {
+pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     info!(context, "Requesting URL {}", url);
 
-    match reqwest::blocking::Client::new()
-        .get(url)
-        .send()
-        .and_then(|res| res.text())
-    {
+    match surf::get(url).recv_string().await {
         Ok(res) => Ok(res),
         Err(err) => {
             info!(context, "Can\'t read URL {}", url);

src/constants.rs

@@ -1,7 +1,6 @@
 //! # Constants
 #![allow(dead_code)]
 
-use deltachat_derive::*;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
@@ -25,12 +24,11 @@ const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
     Eq,
     FromPrimitive,
     ToPrimitive,
-    FromSql,
-    ToSql,
     Serialize,
     Deserialize,
+    sqlx::Type,
 )]
-#[repr(u8)]
+#[repr(i32)]
 pub enum Blocked {
     Not = 0,
     Manually = 1,
@@ -43,8 +41,8 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
-#[repr(u8)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
+#[repr(i32)]
 pub enum ShowEmails {
     Off = 0,
     AcceptedContacts = 1,
@@ -57,7 +55,20 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
+#[repr(i32)]
+pub enum MediaQuality {
+    Balanced = 0,
+    Worse = 1,
+}
+
+impl Default for MediaQuality {
+    fn default() -> Self {
+        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
+    }
+}
+
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -114,13 +125,12 @@ pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
     Eq,
     FromPrimitive,
     ToPrimitive,
-    FromSql,
-    ToSql,
     IntoStaticStr,
     Serialize,
     Deserialize,
+    sqlx::Type,
 )]
-#[repr(u32)]
+#[repr(i32)]
 pub enum Chattype {
     Undefined = 0,
     Single = 100,
@@ -214,6 +224,10 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
+// max. width/height of images
+pub const BALANCED_IMAGE_SIZE: u32 = 1280;
+pub const WORSE_IMAGE_SIZE: u32 = 640;
+
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
@@ -226,10 +240,9 @@ pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
     Eq,
     FromPrimitive,
     ToPrimitive,
-    FromSql,
-    ToSql,
     Serialize,
     Deserialize,
+    sqlx::Type,
 )]
 #[repr(i32)]
 pub enum Viewtype {

src/context.rs

@@ -2,8 +2,10 @@
 
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
-use std::path::{Path, PathBuf};
-use std::sync::{Arc, Condvar, Mutex, RwLock};
+use std::ops::Deref;
+
+use async_std::path::{Path, PathBuf};
+use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
 
 use crate::chat::*;
 use crate::config::Config;
@@ -11,61 +13,58 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::dc_tools::duration_to_str;
 use crate::error::*;
-use crate::events::Event;
-use crate::imap::*;
-use crate::job::*;
-use crate::job_thread::JobThread;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::events::{Event, EventEmitter, Events};
+use crate::job::{self, Action};
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
 use crate::message::{self, Message, MessengerMessage, MsgId};
 use crate::param::Params;
-use crate::smtp::Smtp;
+use crate::scheduler::Scheduler;
 use crate::sql::Sql;
 use std::time::SystemTime;
 
-/// Callback function type for [Context]
-///
-/// # Parameters
-///
-/// * `context` - The context object as returned by [Context::new].
-/// * `event` - One of the [Event] items.
-/// * `data1` - Depends on the event parameter, see [Event].
-/// * `data2` - Depends on the event parameter, see [Event].
-pub type ContextCallback = dyn Fn(&Context, Event) -> () + Send + Sync;
-
-#[derive(DebugStub)]
+#[derive(Clone, Debug)]
 pub struct Context {
+    pub(crate) inner: Arc<InnerContext>,
+}
+
+impl Deref for Context {
+    type Target = InnerContext;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+#[derive(Debug)]
+pub struct InnerContext {
     /// Database file path
-    dbfile: PathBuf,
+    pub(crate) dbfile: PathBuf,
     /// Blob directory path
-    blobdir: PathBuf,
-    pub sql: Sql,
-    pub perform_inbox_jobs_needed: Arc<RwLock<bool>>,
-    pub probe_imap_network: Arc<RwLock<bool>>,
-    pub inbox_thread: Arc<RwLock<JobThread>>,
-    pub sentbox_thread: Arc<RwLock<JobThread>>,
-    pub mvbox_thread: Arc<RwLock<JobThread>>,
-    pub smtp: Arc<Mutex<Smtp>>,
-    pub smtp_state: Arc<(Mutex<SmtpState>, Condvar)>,
-    pub oauth2_critical: Arc<Mutex<()>>,
-    #[debug_stub = "Callback"]
-    cb: Box<ContextCallback>,
-    pub os_name: Option<String>,
-    pub cmdline_sel_chat_id: Arc<RwLock<ChatId>>,
-    pub(crate) bob: Arc<RwLock<BobStatus>>,
-    pub last_smeared_timestamp: RwLock<i64>,
-    pub running_state: Arc<RwLock<RunningState>>,
+    pub(crate) blobdir: PathBuf,
+    pub(crate) sql: Sql,
+    pub(crate) os_name: Option<String>,
+    pub(crate) bob: RwLock<BobStatus>,
+    pub(crate) last_smeared_timestamp: RwLock<i64>,
+    pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
-    pub generating_key_mutex: Mutex<()>,
-    pub translated_stockstrings: RwLock<HashMap<usize, String>>,
+    pub(crate) generating_key_mutex: Mutex<()>,
+    /// Mutex to enforce only a single running oauth2 is running.
+    pub(crate) oauth2_mutex: Mutex<()>,
+    pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
+    pub(crate) events: Events,
+
+    pub(crate) scheduler: RwLock<Scheduler>,
+
     creation_time: SystemTime,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug)]
 pub struct RunningState {
     pub ongoing_running: bool,
     shall_stop_ongoing: bool,
+    cancel_sender: Option<Sender<()>>,
 }
 
 /// Return some info about deltachat-core
@@ -77,7 +76,11 @@ pub struct RunningState {
 pub fn get_info() -> BTreeMap<&'static str, String> {
     let mut res = BTreeMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
-    res.insert("sqlite_version", rusqlite::version().to_string());
+
+    let version =
+        String::from_utf8(libsqlite3_sys::SQLITE_VERSION.to_vec()).expect("invalid version");
+    res.insert("sqlite_version", version);
+
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
     res.insert("level", "awesome".into());
     res
@@ -85,73 +88,91 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context.
-    pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
-        pretty_env_logger::try_init_timed().ok();
-
+    pub async fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         let blobdir = dbfile.with_file_name(blob_fname);
-        if !blobdir.exists() {
-            std::fs::create_dir_all(&blobdir)?;
+        if !blobdir.exists().await {
+            async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(cb, os_name, dbfile, blobdir)
+        Context::with_blobdir(os_name, dbfile, blobdir).await
     }
 
-    pub fn with_blobdir(
-        cb: Box<ContextCallback>,
+    pub async fn with_blobdir(
         os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
     ) -> Result<Context> {
         ensure!(
-            blobdir.is_dir(),
+            blobdir.is_dir().await,
             "Blobdir does not exist: {}",
             blobdir.display()
         );
-        let ctx = Context {
+
+        let inner = InnerContext {
             blobdir,
             dbfile,
-            cb,
             os_name: Some(os_name),
-            running_state: Arc::new(RwLock::new(Default::default())),
+            running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
-            smtp: Arc::new(Mutex::new(Smtp::new())),
-            smtp_state: Arc::new((Mutex::new(Default::default()), Condvar::new())),
-            oauth2_critical: Arc::new(Mutex::new(())),
-            bob: Arc::new(RwLock::new(Default::default())),
+            bob: RwLock::new(Default::default()),
             last_smeared_timestamp: RwLock::new(0),
-            cmdline_sel_chat_id: Arc::new(RwLock::new(ChatId::new(0))),
-            inbox_thread: Arc::new(RwLock::new(JobThread::new(
-                "INBOX",
-                "configured_inbox_folder",
-                Imap::new(),
-            ))),
-            sentbox_thread: Arc::new(RwLock::new(JobThread::new(
-                "SENTBOX",
-                "configured_sentbox_folder",
-                Imap::new(),
-            ))),
-            mvbox_thread: Arc::new(RwLock::new(JobThread::new(
-                "MVBOX",
-                "configured_mvbox_folder",
-                Imap::new(),
-            ))),
-            probe_imap_network: Arc::new(RwLock::new(false)),
-            perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
+            oauth2_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
+            events: Events::default(),
+            scheduler: RwLock::new(Scheduler::Stopped),
             creation_time: std::time::SystemTime::now(),
         };
 
-        ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, false),
-            "Failed opening sqlite database"
-        );
+        let ctx = Context {
+            inner: Arc::new(inner),
+        };
+
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
 
+    /// Starts the IO scheduler.
+    pub async fn start_io(&self) {
+        info!(self, "starting IO");
+        if self.is_io_running().await {
+            info!(self, "IO is already running");
+            return;
+        }
+
+        {
+            let l = &mut *self.inner.scheduler.write().await;
+            l.start(self.clone()).await;
+        }
+    }
+
+    /// Returns if the IO scheduler is running.
+    pub async fn is_io_running(&self) -> bool {
+        self.inner.is_io_running().await
+    }
+
+    /// Stops the IO scheduler.
+    pub async fn stop_io(&self) {
+        info!(self, "stopping IO");
+        if !self.is_io_running().await {
+            info!(self, "IO is not running");
+            return;
+        }
+
+        self.inner.stop_io().await;
+    }
+
+    /// Returns a reference to the underlying SQL instance.
+    ///
+    /// Warning: this is only here for testing, not part of the public API.
+    #[cfg(feature = "internals")]
+    pub fn sql(&self) -> &Sql {
+        &self.inner.sql
+    }
+
     /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
         self.dbfile.as_path()
@@ -162,49 +183,57 @@ impl Context {
         self.blobdir.as_path()
     }
 
-    pub fn call_cb(&self, event: Event) {
-        (*self.cb)(self, event);
+    /// Emits a single event.
+    pub fn emit_event(&self, event: Event) {
+        self.events.emit(event);
     }
 
-    /*******************************************************************************
-     * Ongoing process allocation/free/check
-     ******************************************************************************/
+    /// Get the next queued event.
+    pub fn get_event_emitter(&self) -> EventEmitter {
+        self.events.get_emitter()
+    }
 
-    pub fn alloc_ongoing(&self) -> bool {
-        if self.has_ongoing() {
-            warn!(self, "There is already another ongoing process running.",);
+    // Ongoing process allocation/free/check
 
-            false
-        } else {
-            let s_a = self.running_state.clone();
-            let mut s = s_a.write().unwrap();
-
-            s.ongoing_running = true;
-            s.shall_stop_ongoing = false;
-
-            true
+    pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
+        if self.has_ongoing().await {
+            bail!("There is already another ongoing process running.");
         }
+
+        let s_a = &self.running_state;
+        let mut s = s_a.write().await;
+
+        s.ongoing_running = true;
+        s.shall_stop_ongoing = false;
+        let (sender, receiver) = channel(1);
+        s.cancel_sender = Some(sender);
+
+        Ok(receiver)
     }
 
-    pub fn free_ongoing(&self) {
-        let s_a = self.running_state.clone();
-        let mut s = s_a.write().unwrap();
+    pub async fn free_ongoing(&self) {
+        let s_a = &self.running_state;
+        let mut s = s_a.write().await;
 
         s.ongoing_running = false;
         s.shall_stop_ongoing = true;
+        s.cancel_sender.take();
     }
 
-    pub fn has_ongoing(&self) -> bool {
-        let s_a = self.running_state.clone();
-        let s = s_a.read().unwrap();
+    pub async fn has_ongoing(&self) -> bool {
+        let s_a = &self.running_state;
+        let s = s_a.read().await;
 
         s.ongoing_running || !s.shall_stop_ongoing
     }
 
     /// Signal an ongoing process to stop.
-    pub fn stop_ongoing(&self) {
-        let s_a = self.running_state.clone();
-        let mut s = s_a.write().unwrap();
+    pub async fn stop_ongoing(&self) {
+        let s_a = &self.running_state;
+        let mut s = s_a.write().await;
+        if let Some(cancel) = s.cancel_sender.take() {
+            cancel.send(()).await;
+        }
 
         if s.ongoing_running && !s.shall_stop_ongoing {
             info!(self, "Signaling the ongoing process to stop ASAP.",);
@@ -214,68 +243,71 @@ impl Context {
         };
     }
 
-    pub fn shall_stop_ongoing(&self) -> bool {
-        self.running_state
-            .clone()
-            .read()
-            .unwrap()
-            .shall_stop_ongoing
+    pub async fn shall_stop_ongoing(&self) -> bool {
+        self.running_state.read().await.shall_stop_ongoing
     }
 
     /*******************************************************************************
      * UI chat/message related API
      ******************************************************************************/
 
-    pub fn get_info(&self) -> BTreeMap<&'static str, String> {
+    pub async fn get_info(&self) -> BTreeMap<&'static str, String> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "");
-        let l2 = LoginParam::from_database(self, "configured_");
-        let displayname = self.get_config(Config::Displayname);
-        let chats = get_chat_cnt(self) as usize;
-        let real_msgs = message::get_real_msg_cnt(self) as usize;
-        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self) as usize;
-        let contacts = Contact::get_real_cnt(self) as usize;
-        let is_configured = self.get_config_int(Config::Configured);
+        let l = LoginParam::from_database(self, "").await;
+        let l2 = LoginParam::from_database(self, "configured_").await;
+        let displayname = self.get_config(Config::Displayname).await;
+        let chats = get_chat_cnt(self).await as usize;
+        let real_msgs = message::get_real_msg_cnt(self).await as usize;
+        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self).await as usize;
+        let contacts = Contact::get_real_cnt(self).await as usize;
+        let is_configured = self.get_config_int(Config::Configured).await;
         let dbversion = self
             .sql
-            .get_raw_config_int(self, "dbversion")
+            .get_raw_config_int("dbversion")
+            .await
             .unwrap_or_default();
+        let journal_mode = self
+            .sql
+            .query_value("PRAGMA journal_mode;", paramsx![])
+            .await
+            .unwrap_or_else(|_| "unknown".to_string());
+        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await;
+        let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await;
+        let bcc_self = self.get_config_int(Config::BccSelf).await;
 
-        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
-        let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
-        let bcc_self = self.get_config_int(Config::BccSelf);
+        let prv_key_cnt: Option<i32> = self
+            .sql
+            .query_value("SELECT COUNT(*) FROM keypairs;", paramsx![])
+            .await
+            .ok();
 
-        let prv_key_cnt: Option<isize> =
-            self.sql
-                .query_get_value(self, "SELECT COUNT(*) FROM keypairs;", rusqlite::NO_PARAMS);
-
-        let pub_key_cnt: Option<isize> = self.sql.query_get_value(
-            self,
-            "SELECT COUNT(*) FROM acpeerstates;",
-            rusqlite::NO_PARAMS,
-        );
-
-        let fingerprint_str = match SignedPublicKey::load_self(self) {
-            Ok(key) => Key::from(key).fingerprint(),
+        let pub_key_cnt: Option<i32> = self
+            .sql
+            .query_value("SELECT COUNT(*) FROM acpeerstates;", paramsx![])
+            .await
+            .ok();
+        let fingerprint_str = match SignedPublicKey::load_self(self).await {
+            Ok(key) => key.fingerprint().hex(),
             Err(err) => format!("<key failure: {}>", err),
         };
 
-        let inbox_watch = self.get_config_int(Config::InboxWatch);
-        let sentbox_watch = self.get_config_int(Config::SentboxWatch);
-        let mvbox_watch = self.get_config_int(Config::MvboxWatch);
-        let mvbox_move = self.get_config_int(Config::MvboxMove);
+        let inbox_watch = self.get_config_int(Config::InboxWatch).await;
+        let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
+        let mvbox_move = self.get_config_int(Config::MvboxMove).await;
         let folders_configured = self
             .sql
-            .get_raw_config_int(self, "folders_configured")
+            .get_raw_config_int("folders_configured")
+            .await
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
+            .get_config(Config::ConfiguredSentboxFolder)
+            .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
+            .get_config(Config::ConfiguredMvboxFolder)
+            .await
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
@@ -285,11 +317,13 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
+        res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
             "selfavatar",
             self.get_config(Config::Selfavatar)
+                .await
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
@@ -321,39 +355,32 @@ impl Context {
         res
     }
 
-    pub fn get_fresh_msgs(&self) -> Vec<MsgId> {
-        let show_deaddrop = 0;
+    pub async fn get_fresh_msgs(&self) -> Vec<MsgId> {
+        let show_deaddrop: i32 = 0;
         self.sql
-            .query_map(
-                concat!(
-                    "SELECT m.id",
-                    " FROM msgs m",
-                    " LEFT JOIN contacts ct",
-                    "        ON m.from_id=ct.id",
-                    " LEFT JOIN chats c",
-                    "        ON m.chat_id=c.id",
-                    " WHERE m.state=?",
-                    "   AND m.hidden=0",
-                    "   AND m.chat_id>?",
-                    "   AND ct.blocked=0",
-                    "   AND (c.blocked=0 OR c.blocked=?)",
-                    " ORDER BY m.timestamp DESC,m.id DESC;"
-                ),
-                &[10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
-                |row| row.get::<_, MsgId>(0),
-                |rows| {
-                    let mut ret = Vec::new();
-                    for row in rows {
-                        ret.push(row?);
-                    }
-                    Ok(ret)
-                },
+            .query_values(
+                r#"
+SELECT m.id
+ FROM msgs m
+ LEFT JOIN contacts ct
+        ON m.from_id=ct.id
+ LEFT JOIN chats c
+        ON m.chat_id=c.id
+ WHERE m.state=?
+   AND m.hidden=0
+   AND m.chat_id>?
+   AND ct.blocked=0
+   AND (c.blocked=0 OR c.blocked=?)
+ ORDER BY m.timestamp DESC,m.id DESC;
+"#,
+                paramsx![10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
             )
+            .await
             .unwrap_or_default()
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -362,82 +389,64 @@ impl Context {
         let strLikeBeg = format!("{}%", real_query);
 
         let query = if !chat_id.is_unset() {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " WHERE m.chat_id=?",
-                "   AND m.hidden=0",
-                "   AND ct.blocked=0",
-                "   AND (txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp,m.id;"
-            )
+            r#"
+SELECT m.id
+ FROM msgs 
+ LEFT JOIN contacts ct
+        ON m.from_id=ct.id
+ WHERE m.chat_id=?
+   AND m.hidden=0
+   AND ct.blocked=0
+   AND (txt LIKE ? OR ct.name LIKE ?)
+ ORDER BY m.timestamp,m.id;
+"#
         } else {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " LEFT JOIN chats c",
-                "        ON m.chat_id=c.id",
-                " WHERE m.chat_id>9",
-                "   AND m.hidden=0",
-                "   AND (c.blocked=0 OR c.blocked=?)",
-                "   AND ct.blocked=0",
-                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp DESC,m.id DESC;"
-            )
+            r#"
+SELECT m.id
+ FROM msgs m
+ LEFT JOIN contacts ct
+        ON m.from_id=ct.id
+ LEFT JOIN chats c
+        ON m.chat_id=c.id
+ WHERE m.chat_id>9
+   AND m.hidden=0
+   AND (c.blocked=0 OR c.blocked=?)
+   AND ct.blocked=0
+   AND (m.txt LIKE ? OR ct.name LIKE ?)
+ ORDER BY m.timestamp DESC,m.id DESC;
+"#
         };
 
         self.sql
-            .query_map(
-                query,
-                params![chat_id, &strLikeInText, &strLikeBeg],
-                |row| row.get::<_, MsgId>("id"),
-                |rows| {
-                    let mut ret = Vec::new();
-                    for id in rows {
-                        ret.push(id?);
-                    }
-                    Ok(ret)
-                },
-            )
+            .query_values(query, paramsx![chat_id, strLikeInText, strLikeBeg])
+            .await
             .unwrap_or_default()
     }
 
-    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        folder_name.as_ref() == "INBOX"
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self.sql.get_raw_config(self, "configured_sentbox_folder");
-        if let Some(name) = sentbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+    pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self.sql.get_raw_config(self, "configured_mvbox_folder");
-
-        if let Some(name) = mvbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+    pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove) {
+    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
+        if !self.get_config_bool(Config::MvboxMove).await {
             return;
         }
 
-        if self.is_mvbox(folder) {
+        if self.is_mvbox(folder).await {
             return;
         }
-        if let Ok(msg) = Message::load_from_db(self, msg_id) {
+        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
             if msg.is_setupmessage() {
                 // do not move setup messages;
                 // there may be a non-delta device that wants to handle it
@@ -447,30 +456,32 @@ impl Context {
             match msg.is_dc_message {
                 MessengerMessage::No => {}
                 MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job_add(
+                    job::add(
                         self,
-                        Action::MoveMsg,
-                        msg.id.to_u32() as i32,
-                        Params::new(),
-                        0,
-                    );
+                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
+                    )
+                    .await;
                 }
             }
         }
     }
 }
 
-impl Drop for Context {
-    fn drop(&mut self) {
-        info!(self, "disconnecting inbox-thread",);
-        self.inbox_thread.read().unwrap().imap.disconnect(self);
-        info!(self, "disconnecting sentbox-thread",);
-        self.sentbox_thread.read().unwrap().imap.disconnect(self);
-        info!(self, "disconnecting mvbox-thread",);
-        self.mvbox_thread.read().unwrap().imap.disconnect(self);
-        info!(self, "disconnecting SMTP");
-        self.smtp.clone().lock().unwrap().disconnect();
-        self.sql.close(self);
+impl InnerContext {
+    async fn is_io_running(&self) -> bool {
+        self.scheduler.read().await.is_running()
+    }
+
+    async fn stop_io(&self) {
+        assert!(self.is_io_running().await, "context is already stopped");
+        let token = {
+            let lock = &*self.scheduler.read().await;
+            lock.pre_stop().await
+        };
+        {
+            let lock = &mut *self.scheduler.write().await;
+            lock.stop(token).await;
+        }
     }
 }
 
@@ -479,6 +490,7 @@ impl Default for RunningState {
         RunningState {
             ongoing_running: false,
             shall_stop_ongoing: true,
+            cancel_sender: None,
         }
     }
 }
@@ -490,28 +502,6 @@ pub(crate) struct BobStatus {
     pub qr_scan: Option<Lot>,
 }
 
-#[derive(Debug, PartialEq)]
-pub(crate) enum PerformJobsNeeded {
-    Not,
-    AtOnce,
-    AvoidDos,
-}
-
-impl Default for PerformJobsNeeded {
-    fn default() -> Self {
-        Self::Not
-    }
-}
-
-#[derive(Default, Debug)]
-pub struct SmtpState {
-    pub idle: bool,
-    pub suspended: bool,
-    pub doing_jobs: bool,
-    pub(crate) perform_jobs_needed: PerformJobsNeeded,
-    pub probe_network: bool,
-}
-
 pub fn get_version_str() -> &'static str {
     &DC_VERSION_STR
 }
@@ -522,81 +512,81 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[test]
-    fn test_wrong_db() {
+    #[async_std::test]
+    async fn test_wrong_db() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| ()), "FakeOs".into(), dbfile);
+        let res = Context::new("FakeOs".into(), dbfile.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn test_get_fresh_msgs() {
-        let t = dummy_context();
-        let fresh = t.ctx.get_fresh_msgs();
+    #[async_std::test]
+    async fn test_get_fresh_msgs() {
+        let t = TestContext::new().await;
+        let fresh = t.ctx.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
 
-    #[test]
-    fn test_blobdir_exists() {
+    #[async_std::test]
+    async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
+        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
 
-    #[test]
-    fn test_wrong_blogdir() {
+    #[async_std::test]
+    async fn test_wrong_blogdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile);
+        let res = Context::new("FakeOS".into(), dbfile.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn test_sqlite_parent_not_exists() {
+    #[async_std::test]
+    async fn test_sqlite_parent_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(Box::new(|_, _| ()), "FakeOS".into(), dbfile).unwrap();
+        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
 
-    #[test]
-    fn test_with_empty_blobdir() {
+    #[async_std::test]
+    async fn test_with_empty_blobdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn test_with_blobdir_not_exists() {
+    #[async_std::test]
+    async fn test_with_blobdir_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(Box::new(|_, _| ()), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
         assert!(res.is_err());
     }
 
-    #[test]
-    fn no_crashes_on_context_deref() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn no_crashes_on_context_deref() {
+        let t = TestContext::new().await;
         std::mem::drop(t.ctx);
     }
 
-    #[test]
-    fn test_get_info() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_get_info() {
+        let t = TestContext::new().await;
 
-        let info = t.ctx.get_info();
+        let info = t.ctx.get_info().await;
         assert!(info.get("database_dir").is_some());
     }
 

src/dc_tools.rs

@@ -3,11 +3,12 @@
 
 use core::cmp::{max, min};
 use std::borrow::Cow;
-use std::path::{Path, PathBuf};
+use std::fmt;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
-use std::{fmt, fs};
 
+use async_std::path::{Path, PathBuf};
+use async_std::{fs, io};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
@@ -107,9 +108,9 @@ const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 // returns the currently smeared timestamp,
 // may be used to check if call to dc_create_smeared_timestamp() is needed or not.
 // the returned timestamp MUST NOT be used to be sent out or saved in the database!
-pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
+pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     let mut now = time();
-    let ts = *context.last_smeared_timestamp.read().unwrap();
+    let ts = *context.last_smeared_timestamp.read().await;
     if ts >= now {
         now = ts + 1;
     }
@@ -118,11 +119,11 @@ pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
 }
 
 // returns a timestamp that is guaranteed to be unique.
-pub(crate) fn dc_create_smeared_timestamp(context: &Context) -> i64 {
+pub(crate) async fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
 
-    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
     if ret <= *last_smeared_timestamp {
         ret = *last_smeared_timestamp + 1;
         if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE {
@@ -137,12 +138,12 @@ pub(crate) fn dc_create_smeared_timestamp(context: &Context) -> i64 {
 // creates `count` timestamps that are guaranteed to be unique.
 // the frist created timestamps is returned directly,
 // get the other timestamps just by adding 1..count-1
-pub(crate) fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
+pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
     let now = time();
     let count = count as i64;
     let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count;
 
-    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
     start = max(*last_smeared_timestamp + 1, start);
 
     *last_smeared_timestamp = start + count - 1;
@@ -248,11 +249,8 @@ pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
 ///
 /// If `path` starts with "$BLOBDIR", replaces it with the blobdir path.
 /// Otherwise, returns path as is.
-pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
-    context: &Context,
-    path: P,
-) -> std::path::PathBuf {
-    let p: &std::path::Path = path.as_ref();
+pub(crate) fn dc_get_abs_path<P: AsRef<Path>>(context: &Context, path: P) -> PathBuf {
+    let p: &Path = path.as_ref();
     if let Ok(p) = p.strip_prefix("$BLOBDIR") {
         context.get_blobdir().join(p)
     } else {
@@ -260,20 +258,20 @@ pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) fn dc_get_filebytes(context: &Context, path: impl AsRef<std::path::Path>) -> u64 {
+pub(crate) async fn dc_get_filebytes(context: &Context, path: impl AsRef<Path>) -> u64 {
     let path_abs = dc_get_abs_path(context, &path);
-    match fs::metadata(&path_abs) {
+    match fs::metadata(&path_abs).await {
         Ok(meta) => meta.len() as u64,
         Err(_err) => 0,
     }
 }
 
-pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
+pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists() {
+    if !path_abs.exists().await {
         return false;
     }
-    if !path_abs.is_file() {
+    if !path_abs.is_file().await {
         warn!(
             context,
             "refusing to delete non-file \"{}\".",
@@ -283,9 +281,9 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
     }
 
     let dpath = format!("{}", path.as_ref().to_string_lossy());
-    match fs::remove_file(path_abs) {
+    match fs::remove_file(path_abs).await {
         Ok(_) => {
-            context.call_cb(Event::DeletedBlobFile(dpath));
+            context.emit_event(Event::DeletedBlobFile(dpath));
             true
         }
         Err(err) => {
@@ -295,13 +293,13 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
     }
 }
 
-pub(crate) fn dc_copy_file(
+pub(crate) async fn dc_copy_file(
     context: &Context,
-    src_path: impl AsRef<std::path::Path>,
-    dest_path: impl AsRef<std::path::Path>,
+    src_path: impl AsRef<Path>,
+    dest_path: impl AsRef<Path>,
 ) -> bool {
     let src_abs = dc_get_abs_path(context, &src_path);
-    let mut src_file = match fs::File::open(&src_abs) {
+    let mut src_file = match fs::File::open(&src_abs).await {
         Ok(file) => file,
         Err(err) => {
             warn!(
@@ -319,6 +317,7 @@ pub(crate) fn dc_copy_file(
         .create_new(true)
         .write(true)
         .open(&dest_abs)
+        .await
     {
         Ok(file) => file,
         Err(err) => {
@@ -332,7 +331,7 @@ pub(crate) fn dc_copy_file(
         }
     };
 
-    match std::io::copy(&mut src_file, &mut dest_file) {
+    match io::copy(&mut src_file, &mut dest_file).await {
         Ok(_) => true,
         Err(err) => {
             error!(
@@ -344,20 +343,20 @@ pub(crate) fn dc_copy_file(
             );
             {
                 // Attempt to remove the failed file, swallow errors resulting from that.
-                fs::remove_file(dest_abs).ok();
+                fs::remove_file(dest_abs).await.ok();
             }
             false
         }
     }
 }
 
-pub(crate) fn dc_create_folder(
+pub(crate) async fn dc_create_folder(
     context: &Context,
-    path: impl AsRef<std::path::Path>,
-) -> Result<(), std::io::Error> {
+    path: impl AsRef<Path>,
+) -> Result<(), io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists() {
-        match fs::create_dir_all(path_abs) {
+    if !path_abs.exists().await {
+        match fs::create_dir_all(path_abs).await {
             Ok(_) => Ok(()),
             Err(err) => {
                 warn!(
@@ -375,13 +374,13 @@ pub(crate) fn dc_create_folder(
 }
 
 /// Write a the given content to provied file path.
-pub(crate) fn dc_write_file(
+pub(crate) async fn dc_write_file(
     context: &Context,
     path: impl AsRef<Path>,
     buf: &[u8],
-) -> Result<(), std::io::Error> {
+) -> Result<(), io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    fs::write(&path_abs, buf).map_err(|err| {
+    fs::write(&path_abs, buf).await.map_err(|err| {
         warn!(
             context,
             "Cannot write {} bytes to \"{}\": {}",
@@ -393,13 +392,10 @@ pub(crate) fn dc_write_file(
     })
 }
 
-pub fn dc_read_file<P: AsRef<std::path::Path>>(
-    context: &Context,
-    path: P,
-) -> Result<Vec<u8>, Error> {
+pub async fn dc_read_file<P: AsRef<Path>>(context: &Context, path: P) -> Result<Vec<u8>, Error> {
     let path_abs = dc_get_abs_path(context, &path);
 
-    match fs::read(&path_abs) {
+    match fs::read(&path_abs).await {
         Ok(bytes) => Ok(bytes),
         Err(err) => {
             warn!(
@@ -413,13 +409,31 @@ pub fn dc_read_file<P: AsRef<std::path::Path>>(
     }
 }
 
-pub fn dc_open_file<P: AsRef<std::path::Path>>(
+pub async fn dc_open_file<P: AsRef<Path>>(context: &Context, path: P) -> Result<fs::File, Error> {
+    let path_abs = dc_get_abs_path(context, &path);
+
+    match fs::File::open(&path_abs).await {
+        Ok(bytes) => Ok(bytes),
+        Err(err) => {
+            warn!(
+                context,
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
+            );
+            Err(err.into())
+        }
+    }
+}
+
+pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     context: &Context,
     path: P,
 ) -> Result<std::fs::File, Error> {
-    let path_abs = dc_get_abs_path(context, &path);
+    let p: PathBuf = path.as_ref().into();
+    let path_abs = dc_get_abs_path(context, p);
 
-    match fs::File::open(&path_abs) {
+    match std::fs::File::open(&path_abs) {
         Ok(bytes) => Ok(bytes),
         Err(err) => {
             warn!(
@@ -433,7 +447,7 @@ pub fn dc_open_file<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) fn dc_get_next_backup_path(
+pub(crate) async fn dc_get_next_backup_path(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -446,7 +460,7 @@ pub(crate) fn dc_get_next_backup_path(
     for i in 0..64 {
         let mut path = folder.clone();
         path.push(format!("{}-{}.bak", stem, i));
-        if !path.exists() {
+        if !path.exists().await {
             return Ok(path);
         }
     }
@@ -468,15 +482,6 @@ pub struct InvalidEmailError {
     addr: String,
 }
 
-impl InvalidEmailError {
-    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
-        InvalidEmailError {
-            message: msg.into(),
-            addr: addr.into(),
-        }
-    }
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -516,17 +521,24 @@ impl FromStr for EmailAddress {
 
     /// Performs a dead-simple parse of an email address.
     fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        if input.is_empty() {
-            return Err(InvalidEmailError::new("empty string is not valid", input));
-        }
-        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
-
         let err = |msg: &str| {
             Err(InvalidEmailError {
                 message: msg.to_string(),
                 addr: input.to_string(),
             })
         };
+        if input.is_empty() {
+            return err("empty string is not valid");
+        }
+        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
+
+        if input
+            .chars()
+            .any(|c| c.is_whitespace() || c == '<' || c == '>')
+        {
+            return err("Email must not contain whitespaces, '>' or '<'");
+        }
+
         match &parts[..] {
             [domain, local] => {
                 if local.is_empty() {
@@ -556,14 +568,6 @@ impl FromStr for EmailAddress {
     }
 }
 
-impl rusqlite::types::ToSql for EmailAddress {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-        let val = rusqlite::types::Value::Text(self.to_string());
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
-    }
-}
-
 /// Utility to check if a in the binary represantion of listflags
 /// the bit at position bitindex is 1.
 pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
@@ -760,31 +764,35 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_file_handling() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_file_handling() {
+        let t = TestContext::new().await;
         let context = &t.ctx;
-        let dc_file_exist = |ctx: &Context, fname: &str| {
-            ctx.get_blobdir()
-                .join(Path::new(fname).file_name().unwrap())
-                .exists()
-        };
-
-        assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje"));
-        if dc_file_exist(context, "$BLOBDIR/foobar")
-            || dc_file_exist(context, "$BLOBDIR/dada")
-            || dc_file_exist(context, "$BLOBDIR/foobar.dadada")
-            || dc_file_exist(context, "$BLOBDIR/foobar-folder")
-        {
-            dc_delete_file(context, "$BLOBDIR/foobar");
-            dc_delete_file(context, "$BLOBDIR/dada");
-            dc_delete_file(context, "$BLOBDIR/foobar.dadada");
-            dc_delete_file(context, "$BLOBDIR/foobar-folder");
+        macro_rules! dc_file_exist {
+            ($ctx:expr, $fname:expr) => {
+                $ctx.get_blobdir()
+                    .join(Path::new($fname).file_name().unwrap())
+                    .exists()
+            };
         }
-        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content").is_ok());
-        assert!(dc_file_exist(context, "$BLOBDIR/foobar",));
-        assert!(!dc_file_exist(context, "$BLOBDIR/foobarx"));
-        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar"), 7);
+
+        assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
+        if dc_file_exist!(context, "$BLOBDIR/foobar").await
+            || dc_file_exist!(context, "$BLOBDIR/dada").await
+            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
+            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
+        {
+            dc_delete_file(context, "$BLOBDIR/foobar").await;
+            dc_delete_file(context, "$BLOBDIR/dada").await;
+            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
+            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
+        }
+        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
+            .await
+            .is_ok());
+        assert!(dc_file_exist!(context, "$BLOBDIR/foobar").await);
+        assert!(!dc_file_exist!(context, "$BLOBDIR/foobarx").await);
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar").await, 7);
 
         let abs_path = context
             .get_blobdir()
@@ -792,31 +800,33 @@ mod tests {
             .to_string_lossy()
             .to_string();
 
-        assert!(dc_file_exist(context, &abs_path));
+        assert!(dc_file_exist!(context, &abs_path).await);
 
-        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
 
         // attempting to copy a second time should fail
-        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
 
-        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada",), 7);
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada").await, 7);
 
-        let buf = dc_read_file(context, "$BLOBDIR/dada").unwrap();
+        let buf = dc_read_file(context, "$BLOBDIR/dada").await.unwrap();
 
         assert_eq!(buf.len(), 7);
         assert_eq!(&buf, b"content");
 
-        assert!(dc_delete_file(context, "$BLOBDIR/foobar"));
-        assert!(dc_delete_file(context, "$BLOBDIR/dada"));
-        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder").is_ok());
-        assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
-        assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
+        assert!(dc_delete_file(context, "$BLOBDIR/foobar").await);
+        assert!(dc_delete_file(context, "$BLOBDIR/dada").await);
+        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder")
+            .await
+            .is_ok());
+        assert!(dc_file_exist!(context, "$BLOBDIR/foobar-folder").await);
+        assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder").await);
 
         let fn0 = "$BLOBDIR/data.data";
-        assert!(dc_write_file(context, &fn0, b"content").is_ok());
+        assert!(dc_write_file(context, &fn0, b"content").await.is_ok());
 
-        assert!(dc_delete_file(context, &fn0));
-        assert!(!dc_file_exist(context, &fn0));
+        assert!(dc_delete_file(context, &fn0).await);
+        assert!(!dc_file_exist!(context, &fn0).await);
     }
 
     #[test]
@@ -833,15 +843,15 @@ mod tests {
         assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
     }
 
-    #[test]
-    fn test_create_smeared_timestamp() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_smeared_timestamp() {
+        let t = TestContext::new().await;
         assert_ne!(
-            dc_create_smeared_timestamp(&t.ctx),
-            dc_create_smeared_timestamp(&t.ctx)
+            dc_create_smeared_timestamp(&t.ctx).await,
+            dc_create_smeared_timestamp(&t.ctx).await
         );
         assert!(
-            dc_create_smeared_timestamp(&t.ctx)
+            dc_create_smeared_timestamp(&t.ctx).await
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -849,17 +859,17 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_create_smeared_timestamps() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_smeared_timestamps() {
+        let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
-        let next = dc_smeared_time(&t.ctx);
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
+        let next = dc_smeared_time(&t.ctx).await;
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
-        let next = dc_smeared_time(&t.ctx);
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
+        let next = dc_smeared_time(&t.ctx).await;
         assert!((start + count - 1) < next);
     }
 

src/e2ee.rs

@@ -9,8 +9,9 @@ use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
 use crate::error::*;
-use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
+use crate::headerdef::HeaderDef;
+use crate::headerdef::HeaderDefMap;
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -24,18 +25,18 @@ pub struct EncryptHelper {
 }
 
 impl EncryptHelper {
-    pub fn new(context: &Context) -> Result<EncryptHelper> {
+    pub async fn new(context: &Context) -> Result<EncryptHelper> {
         let prefer_encrypt =
-            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled))
+            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await)
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr) {
+        let addr = match context.get_config(Config::ConfiguredAddr).await {
             None => {
                 bail!("addr not configured!");
             }
             Some(addr) => addr,
         };
 
-        let public_key = SignedPublicKey::load_self(context)?;
+        let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -85,45 +86,44 @@ impl EncryptHelper {
     }
 
     /// Tries to encrypt the passed in `mail`.
-    pub fn encrypt(
-        &mut self,
+    pub async fn encrypt(
+        self,
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: &[(Option<Peerstate>, &str)],
+        peerstates: Vec<(Option<Peerstate>, &str)>,
     ) -> Result<String> {
-        let mut keyring = Keyring::default();
+        let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
         for (peerstate, addr) in peerstates
-            .iter()
-            .filter_map(|(state, addr)| state.as_ref().map(|s| (s, addr)))
+            .into_iter()
+            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate.peek_key(min_verified).ok_or_else(|| {
+            let key = peerstate.take_key(min_verified).ok_or_else(|| {
                 format_err!("proper enc-key for {} missing, cannot encrypt", addr)
             })?;
-            keyring.add_ref(key);
+            keyring.add(key);
         }
-        let public_key = Key::from(self.public_key.clone());
-        keyring.add_ref(&public_key);
-        let sign_key = Key::from(SignedSecretKey::load_self(context)?);
+        keyring.add(self.public_key.clone());
+        let sign_key = SignedSecretKey::load_self(context).await?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
-        let ctext = pgp::pk_encrypt(&raw_message, &keyring, Some(&sign_key))?;
+        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key)).await?;
 
         Ok(ctext)
     }
 }
 
-pub fn try_decrypt(
+pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
     let from = mail
         .headers
-        .get_header_value(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
+        .get_header(HeaderDef::From_)
+        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -132,56 +132,51 @@ pub fn try_decrypt(
     let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
 
     if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &context.sql, &from);
+        peerstate = Peerstate::from_addr(context, &from).await.ok();
 
         if let Some(ref mut peerstate) = peerstate {
             if let Some(ref header) = autocryptheader {
                 peerstate.apply_header(&header, message_time);
-                peerstate.save_to_db(&context.sql, false)?;
+                peerstate.save_to_db(&context.sql, false).await?;
             } else if message_time > peerstate.last_seen_autocrypt && !contains_report(mail) {
                 peerstate.degrade_encryption(message_time);
-                peerstate.save_to_db(&context.sql, false)?;
+                peerstate.save_to_db(&context.sql, false).await?;
             }
         } else if let Some(ref header) = autocryptheader {
-            let p = Peerstate::from_header(context, header, message_time);
-            p.save_to_db(&context.sql, true)?;
+            let p = Peerstate::from_header(header, message_time);
+            p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
     }
 
     /* possibly perform decryption */
-    let mut private_keyring = Keyring::default();
-    let mut public_keyring_for_validate = Keyring::default();
-    let mut out_mail = None;
+    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
+    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
-    let self_addr = context.get_config(Config::ConfiguredAddr);
 
-    if let Some(self_addr) = self_addr {
-        if private_keyring.load_self_private_for_decrypting(context, self_addr, &context.sql) {
-            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-                peerstate = Peerstate::from_addr(&context, &context.sql, &from);
-            }
-            if let Some(ref peerstate) = peerstate {
-                if peerstate.degrade_event.is_some() {
-                    handle_degrade_event(context, &peerstate)?;
-                }
-                if let Some(ref key) = peerstate.gossip_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-                if let Some(ref key) = peerstate.public_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-            }
-
-            out_mail = decrypt_if_autocrypt_message(
-                context,
-                mail,
-                &private_keyring,
-                &public_keyring_for_validate,
-                &mut signatures,
-            )?;
+    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
+        peerstate = Peerstate::from_addr(&context, &from).await.ok();
+    }
+    if let Some(peerstate) = peerstate {
+        if peerstate.degrade_event.is_some() {
+            handle_degrade_event(context, &peerstate).await?;
+        }
+        if let Some(key) = peerstate.gossip_key {
+            public_keyring_for_validate.add(key);
+        }
+        if let Some(key) = peerstate.public_key {
+            public_keyring_for_validate.add(key);
         }
     }
+
+    let out_mail = decrypt_if_autocrypt_message(
+        context,
+        mail,
+        private_keyring,
+        public_keyring_for_validate,
+        &mut signatures,
+    )
+    .await?;
     Ok((out_mail, signatures))
 }
 
@@ -212,12 +207,12 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
     Ok(&mail.subparts[1])
 }
 
-fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
     mail: &ParsedMail<'a>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     //  The returned bool is true if we detected an Autocrypt-encrypted
     // message and successfully decrypted it. Decryption then modifies the
@@ -236,21 +231,20 @@ fn decrypt_if_autocrypt_message<'a>(
     info!(context, "Detected Autocrypt-mime message");
 
     decrypt_part(
-        context,
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
         ret_valid_signatures,
     )
+    .await
 }
 
 /// Returns Ok(None) if nothing encrypted was found.
-fn decrypt_part(
-    _context: &Context,
+async fn decrypt_part(
     mail: &ParsedMail<'_>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
@@ -259,11 +253,12 @@ fn decrypt_part(
         ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
         let plain = pgp::pk_decrypt(
-            &data,
-            &private_keyring,
-            &public_keyring_for_validate,
+            data,
+            private_keyring,
+            public_keyring_for_validate,
             Some(ret_valid_signatures),
-        )?;
+        )
+        .await?;
 
         ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
         return Ok(Some(plain));
@@ -307,14 +302,17 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 // TODO, remove this once deltachat::key::Key no longer exists.
-pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
-        format_err!(concat!(
-            "Failed to get self address, ",
-            "cannot ensure secret key if not configured."
-        ))
-    })?;
-    SignedPublicKey::load_self(context)?;
+pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
+    let self_addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await
+        .ok_or_else(|| {
+            format_err!(concat!(
+                "Failed to get self address, ",
+                "cannot ensure secret key if not configured."
+            ))
+        })?;
+    SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }
 
@@ -327,17 +325,17 @@ mod tests {
     mod ensure_secret_key_exists {
         use super::*;
 
-        #[test]
-        fn test_prexisting() {
-            let t = dummy_context();
-            let test_addr = configure_alice_keypair(&t.ctx);
-            assert_eq!(ensure_secret_key_exists(&t.ctx).unwrap(), test_addr);
+        #[async_std::test]
+        async fn test_prexisting() {
+            let t = TestContext::new().await;
+            let test_addr = t.configure_alice().await;
+            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
         }
 
-        #[test]
-        fn test_not_configured() {
-            let t = dummy_context();
-            assert!(ensure_secret_key_exists(&t.ctx).is_err());
+        #[async_std::test]
+        async fn test_not_configured() {
+            let t = TestContext::new().await;
+            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
         }
     }
 

src/error.rs

@@ -2,13 +2,6 @@
 
 pub use anyhow::{bail, ensure, format_err, Error, Result};
 
-// #[fail(display = "Invalid Message ID.")]
-// InvalidMsgId,
-// #[fail(display = "Watch folder not found {:?}", _0)]
-// WatchFolderNotFound(String),
-// #[fail(display = "Not Configured")]
-// NotConfigured,
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -1,12 +1,65 @@
 //! # Events specification
 
-use std::path::PathBuf;
-
+use async_std::path::PathBuf;
+use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
 use crate::message::MsgId;
 
+#[derive(Debug)]
+pub struct Events {
+    receiver: Receiver<Event>,
+    sender: Sender<Event>,
+}
+
+impl Default for Events {
+    fn default() -> Self {
+        let (sender, receiver) = channel(1_000);
+
+        Self { receiver, sender }
+    }
+}
+
+impl Events {
+    pub fn emit(&self, event: Event) {
+        match self.sender.try_send(event) {
+            Ok(()) => {}
+            Err(TrySendError::Full(event)) => {
+                // when we are full, we pop remove the oldest event and push on the new one
+                let _ = self.receiver.try_recv();
+
+                // try again
+                self.emit(event);
+            }
+            Err(TrySendError::Disconnected(_)) => {
+                unreachable!("unable to emit event, channel disconnected");
+            }
+        }
+    }
+
+    /// Retrieve the event emitter.
+    pub fn get_emitter(&self) -> EventEmitter {
+        EventEmitter(self.receiver.clone())
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EventEmitter(Receiver<Event>);
+
+impl EventEmitter {
+    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
+    pub fn recv_sync(&self) -> Option<Event> {
+        async_std::task::block_on(self.recv())
+    }
+
+    /// Blocking async recv of an event. Return `None` if the `Sender` has been droped.
+    pub async fn recv(&self) -> Option<Event> {
+        // TODO: change once we can use async channels internally.
+        self.0.recv().await.ok()
+    }
+}
+
 impl Event {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {

src/headerdef.rs

@@ -21,6 +21,7 @@ pub enum HeaderDef {
     References,
     InReplyTo,
     Precedence,
+    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -53,12 +54,16 @@ impl HeaderDef {
 
 pub trait HeaderDefMap {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
 }
 
 impl HeaderDefMap for [MailHeader<'_>] {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
         self.get_first_value(headerdef.get_headername())
     }
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
+        self.get_first_header(headerdef.get_headername())
+    }
 }
 
 #[cfg(test)]

src/imap/client.rs

@@ -1,28 +1,89 @@
+use std::ops::{Deref, DerefMut};
+
 use async_imap::{
     error::{Error as ImapError, Result as ImapResult},
     Client as ImapClient,
 };
-use async_native_tls::TlsStream;
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
+use crate::login_param::dc_build_tls;
+
+use super::session::SessionStream;
 
 #[derive(Debug)]
-pub(crate) enum Client {
-    Secure(ImapClient<TlsStream<TcpStream>>),
-    Insecure(ImapClient<TcpStream>),
+pub(crate) struct Client {
+    is_secure: bool,
+    inner: ImapClient<Box<dyn SessionStream>>,
+}
+
+impl Deref for Client {
+    type Target = ImapClient<Box<dyn SessionStream>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl DerefMut for Client {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
+    }
 }
 
 impl Client {
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> std::result::Result<Session, (ImapError, Self)> {
+        let Client { inner, is_secure } = self;
+        let session = inner
+            .login(username, password)
+            .await
+            .map_err(|(err, client)| {
+                (
+                    err,
+                    Client {
+                        is_secure,
+                        inner: client,
+                    },
+                )
+            })?;
+        Ok(Session { inner: session })
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> std::result::Result<Session, (ImapError, Self)> {
+        let Client { inner, is_secure } = self;
+        let session =
+            inner
+                .authenticate(auth_type, authenticator)
+                .await
+                .map_err(|(err, client)| {
+                    (
+                        err,
+                        Client {
+                            is_secure,
+                            inner: client,
+                        },
+                    )
+                })?;
+        Ok(Session { inner: session })
+    }
+
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        certificate_checks: CertificateChecks,
+        strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
-        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
+        let tls = dc_build_tls(strict_tls);
+        let tls_stream: Box<dyn SessionStream> =
+            Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
         if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
             client.debug = true;
@@ -33,11 +94,14 @@ impl Client {
             .await
             .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
-        Ok(Client::Secure(client))
+        Ok(Client {
+            is_secure: true,
+            inner: client,
+        })
     }
 
     pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
+        let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
         if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
@@ -48,57 +112,28 @@ impl Client {
             .await
             .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
-        Ok(Client::Insecure(client))
+        Ok(Client {
+            is_secure: false,
+            inner: client,
+        })
     }
 
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
-        match self {
-            Client::Insecure(client) => {
-                let tls = dc_build_tls(certificate_checks);
-                let client_sec = client.secure(domain, tls).await?;
+    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
+        if self.is_secure {
+            Ok(self)
+        } else {
+            let Client { mut inner, .. } = self;
+            let tls = dc_build_tls(strict_tls);
+            inner.run_command_and_check_ok("STARTTLS", None).await?;
 
-                Ok(Client::Secure(client_sec))
-            }
-            // Nothing to do
-            Client::Secure(_) => Ok(self),
-        }
-    }
+            let stream = inner.into_inner();
+            let ssl_stream = tls.connect(domain.as_ref(), stream).await?;
+            let boxed: Box<dyn SessionStream> = Box::new(ssl_stream);
 
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
-        self,
-        auth_type: S,
-        authenticator: &A,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
-        self,
-        username: U,
-        password: P,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
+            Ok(Client {
+                is_secure: true,
+                inner: ImapClient::new(boxed),
+            })
         }
     }
 }

src/imap/idle.rs

@@ -1,14 +1,10 @@
 use super::Imap;
 
-use async_imap::extensions::idle::{Handle as ImapIdleHandle, IdleResponse};
-use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
+use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
-use async_std::task;
-use std::sync::atomic::Ordering;
 use std::time::{Duration, SystemTime};
 
-use crate::context::Context;
+use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::select_folder;
 use super::session::Session;
@@ -33,255 +29,203 @@ pub enum Error {
     SetupHandleError(#[from] super::Error),
 }
 
-#[derive(Debug)]
-pub(crate) enum IdleHandle {
-    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
-    Insecure(ImapIdleHandle<TcpStream>),
-}
-
-impl Session {
-    pub fn idle(self) -> IdleHandle {
-        match self {
-            Session::Secure(i) => {
-                let h = i.idle();
-                IdleHandle::Secure(h)
-            }
-            Session::Insecure(i) => {
-                let h = i.idle();
-                IdleHandle::Insecure(h)
-            }
-        }
-    }
-}
-
 impl Imap {
     pub fn can_idle(&self) -> bool {
-        task::block_on(async move { self.config.read().await.can_idle })
+        self.config.can_idle
     }
 
-    pub fn idle(&self, context: &Context, watch_folder: Option<String>) -> Result<()> {
-        task::block_on(async move {
-            if !self.can_idle() {
-                return Err(Error::IdleAbilityMissing);
+    pub async fn idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> Result<InterruptInfo> {
+        use futures::future::FutureExt;
+
+        if !self.can_idle() {
+            return Err(Error::IdleAbilityMissing);
+        }
+        self.setup_handle_if_needed(context).await?;
+
+        self.select_folder(context, watch_folder.clone()).await?;
+
+        let session = self.session.take();
+        let timeout = Duration::from_secs(23 * 60);
+        let mut info = Default::default();
+
+        if let Some(session) = session {
+            let mut handle = session.idle();
+            if let Err(err) = handle.init().await {
+                return Err(Error::IdleProtocolFailed(err));
             }
 
-            self.setup_handle_if_needed(context).await?;
+            let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
 
-            self.select_folder(context, watch_folder.clone()).await?;
-
-            let session = self.session.lock().await.take();
-            let timeout = Duration::from_secs(23 * 60);
-            if let Some(session) = session {
-                match session.idle() {
-                    // BEWARE: If you change the Secure branch you
-                    // typically also need to change the Insecure branch.
-                    IdleHandle::Secure(mut handle) => {
-                        handle.init().await?;
-
-                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
-                        *self.interrupt.lock().await = Some(interrupt);
-
-                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
-                            // interrupt_idle has happened before we
-                            // provided self.interrupt
-                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
-                            std::mem::drop(idle_wait);
-                            info!(context, "Idle wait was skipped");
-                        } else {
-                            info!(context, "Idle entering wait-on-remote state");
-                            match idle_wait.await {
-                                Ok(IdleResponse::NewData(_)) => {
-                                    info!(context, "Idle has NewData");
-                                }
-                                // TODO: idle_wait does not distinguish manual interrupts
-                                // from Timeouts if we would know it's a Timeout we could bail
-                                // directly and reconnect .
-                                Ok(IdleResponse::Timeout) => {
-                                    info!(context, "Idle-wait timeout or interruption");
-                                }
-                                Ok(IdleResponse::ManualInterrupt) => {
-                                    info!(context, "Idle wait was interrupted");
-                                }
-                                Err(err) => {
-                                    warn!(context, "Idle wait errored: {:?}", err);
-                                }
-                            }
-                        }
-                        // if we can't properly terminate the idle
-                        // protocol let's break the connection.
-                        let res =
-                            async_std::future::timeout(Duration::from_secs(15), handle.done())
-                                .await
-                                .map_err(|err| {
-                                    self.trigger_reconnect();
-                                    Error::IdleTimeout(err)
-                                })?;
-
-                        match res {
-                            Ok(session) => {
-                                *self.session.lock().await = Some(Session::Secure(session));
-                            }
-                            Err(err) => {
-                                // if we cannot terminate IDLE it probably
-                                // means that we waited long (with idle_wait)
-                                // but the network went away/changed
-                                self.trigger_reconnect();
-                                return Err(Error::IdleProtocolFailed(err));
-                            }
-                        }
-                    }
-                    IdleHandle::Insecure(mut handle) => {
-                        handle.init().await?;
-
-                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
-                        *self.interrupt.lock().await = Some(interrupt);
-
-                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
-                            // interrupt_idle has happened before we
-                            // provided self.interrupt
-                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
-                            std::mem::drop(idle_wait);
-                            info!(context, "Idle wait was skipped");
-                        } else {
-                            info!(context, "Idle entering wait-on-remote state");
-                            match idle_wait.await {
-                                Ok(IdleResponse::NewData(_)) => {
-                                    info!(context, "Idle has NewData");
-                                }
-                                // TODO: idle_wait does not distinguish manual interrupts
-                                // from Timeouts if we would know it's a Timeout we could bail
-                                // directly and reconnect .
-                                Ok(IdleResponse::Timeout) => {
-                                    info!(context, "Idle-wait timeout or interruption");
-                                }
-                                Ok(IdleResponse::ManualInterrupt) => {
-                                    info!(context, "Idle wait was interrupted");
-                                }
-                                Err(err) => {
-                                    warn!(context, "Idle wait errored: {:?}", err);
-                                }
-                            }
-                        }
-                        // if we can't properly terminate the idle
-                        // protocol let's break the connection.
-                        let res =
-                            async_std::future::timeout(Duration::from_secs(15), handle.done())
-                                .await
-                                .map_err(|err| {
-                                    self.trigger_reconnect();
-                                    Error::IdleTimeout(err)
-                                })?;
-
-                        match res {
-                            Ok(session) => {
-                                *self.session.lock().await = Some(Session::Insecure(session));
-                            }
-                            Err(err) => {
-                                // if we cannot terminate IDLE it probably
-                                // means that we waited long (with idle_wait)
-                                // but the network went away/changed
-                                self.trigger_reconnect();
-                                return Err(Error::IdleProtocolFailed(err));
-                            }
-                        }
-                    }
-                }
+            enum Event {
+                IdleResponse(IdleResponse),
+                Interrupt(InterruptInfo),
             }
 
-            Ok(())
-        })
-    }
-
-    pub(crate) fn fake_idle(&self, context: &Context, watch_folder: Option<String>) {
-        // Idle using polling. This is also needed if we're not yet configured -
-        // in this case, we're waiting for a configure job (and an interrupt).
-        task::block_on(async move {
-            let fake_idle_start_time = SystemTime::now();
-
-            info!(context, "IMAP-fake-IDLEing...");
-
-            let interrupt = stop_token::StopSource::new();
-
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let interval = async_std::stream::interval(Duration::from_secs(60));
-            let mut interrupt_interval = interrupt.stop_token().stop_stream(interval);
-            *self.interrupt.lock().await = Some(interrupt);
-            if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+            if self.skip_next_idle_wait {
                 // interrupt_idle has happened before we
                 // provided self.interrupt
-                self.skip_next_idle_wait.store(false, Ordering::SeqCst);
-                info!(context, "fake-idle wait was skipped");
-            } else {
-                // loop until we are interrupted or if we fetched something
-                while let Some(_) = interrupt_interval.next().await {
-                    // try to connect with proper login params
-                    // (setup_handle_if_needed might not know about them if we
-                    // never successfully connected)
-                    if let Err(err) = self.connect_configured(context) {
-                        warn!(context, "fake_idle: could not connect: {}", err);
-                        continue;
-                    }
-                    if self.config.read().await.can_idle {
-                        // we only fake-idled because network was gone during IDLE, probably
-                        break;
-                    }
-                    info!(context, "fake_idle is connected");
-                    // we are connected, let's see if fetching messages results
-                    // in anything.  If so, we behave as if IDLE had data but
-                    // will have already fetched the messages so perform_*_fetch
-                    // will not find any new.
+                self.skip_next_idle_wait = false;
+                drop(idle_wait);
+                drop(interrupt);
 
-                    if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder).await {
-                            Ok(res) => {
-                                info!(context, "fetch_new_messages returned {:?}", res);
-                                if res {
-                                    break;
-                                }
-                            }
-                            Err(err) => {
-                                error!(context, "could not fetch from folder: {}", err);
-                                self.trigger_reconnect()
-                            }
-                        }
+                info!(context, "Idle wait was skipped");
+            } else {
+                info!(context, "Idle entering wait-on-remote state");
+                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
+                    self.idle_interrupt.recv().map(|probe_network| {
+                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+                    }),
+                );
+
+                match fut.await {
+                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
+                        info!(context, "Idle has NewData");
+                    }
+                    // TODO: idle_wait does not distinguish manual interrupts
+                    // from Timeouts if we would know it's a Timeout we could bail
+                    // directly and reconnect .
+                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                        info!(context, "Idle-wait timeout or interruption");
+                    }
+                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                        info!(context, "Idle wait was interrupted");
+                    }
+                    Ok(Event::Interrupt(i)) => {
+                        info = i;
+                        info!(context, "Idle wait was interrupted");
+                    }
+                    Err(err) => {
+                        warn!(context, "Idle wait errored: {:?}", err);
                     }
                 }
             }
-            self.interrupt.lock().await.take();
 
-            info!(
-                context,
-                "IMAP-fake-IDLE done after {:.4}s",
-                SystemTime::now()
-                    .duration_since(fake_idle_start_time)
-                    .unwrap_or_default()
-                    .as_millis() as f64
-                    / 1000.,
-            );
-        })
+            // if we can't properly terminate the idle
+            // protocol let's break the connection.
+            let res = handle
+                .done()
+                .timeout(Duration::from_secs(15))
+                .await
+                .map_err(|err| {
+                    self.trigger_reconnect();
+                    Error::IdleTimeout(err)
+                })?;
+
+            match res {
+                Ok(session) => {
+                    self.session = Some(Session { inner: session });
+                }
+                Err(err) => {
+                    // if we cannot terminate IDLE it probably
+                    // means that we waited long (with idle_wait)
+                    // but the network went away/changed
+                    self.trigger_reconnect();
+                    return Err(Error::IdleProtocolFailed(err));
+                }
+            }
+        }
+
+        Ok(info)
     }
 
-    pub fn interrupt_idle(&self, context: &Context) {
-        task::block_on(async move {
-            let mut interrupt: Option<stop_token::StopSource> = self.interrupt.lock().await.take();
-            if interrupt.is_none() {
-                // idle wait is not running, signal it needs to skip
-                self.skip_next_idle_wait.store(true, Ordering::SeqCst);
+    pub(crate) async fn fake_idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> InterruptInfo {
+        // Idle using polling. This is also needed if we're not yet configured -
+        // in this case, we're waiting for a configure job (and an interrupt).
 
-                // meanwhile idle-wait may have produced the StopSource
-                interrupt = self.interrupt.lock().await.take();
+        let fake_idle_start_time = SystemTime::now();
+        info!(context, "IMAP-fake-IDLEing...");
+
+        // Do not poll, just wait for an interrupt when no folder is passed in.
+        if watch_folder.is_none() {
+            return self.idle_interrupt.recv().await.unwrap_or_default();
+        }
+
+        let mut info: InterruptInfo = Default::default();
+        if self.skip_next_idle_wait {
+            // interrupt_idle has happened before we
+            // provided self.interrupt
+            self.skip_next_idle_wait = false;
+            info!(context, "fake-idle wait was skipped");
+        } else {
+            // check every minute if there are new messages
+            // TODO: grow sleep durations / make them more flexible
+            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+
+            enum Event {
+                Tick,
+                Interrupt(InterruptInfo),
             }
-            // let's manually drop the StopSource
-            if interrupt.is_some() {
-                // the imap thread provided us a stop token but might
-                // not have entered idle_wait yet, give it some time
-                // for that to happen. XXX handle this without extra wait
-                // https://github.com/deltachat/deltachat-core-rust/issues/925
-                std::thread::sleep(Duration::from_millis(200));
-                info!(context, "low-level: dropping stop-source to interrupt idle");
-                std::mem::drop(interrupt)
-            }
-        });
+            // loop until we are interrupted or if we fetched something
+            info =
+                loop {
+                    use futures::future::FutureExt;
+                    match interval
+                        .next()
+                        .map(|_| Event::Tick)
+                        .race(self.idle_interrupt.recv().map(|probe_network| {
+                            Event::Interrupt(probe_network.unwrap_or_default())
+                        }))
+                        .await
+                    {
+                        Event::Tick => {
+                            // try to connect with proper login params
+                            // (setup_handle_if_needed might not know about them if we
+                            // never successfully connected)
+                            if let Err(err) = self.connect_configured(context).await {
+                                warn!(context, "fake_idle: could not connect: {}", err);
+                                continue;
+                            }
+                            if self.config.can_idle {
+                                // we only fake-idled because network was gone during IDLE, probably
+                                break InterruptInfo::new(false, None);
+                            }
+                            info!(context, "fake_idle is connected");
+                            // we are connected, let's see if fetching messages results
+                            // in anything.  If so, we behave as if IDLE had data but
+                            // will have already fetched the messages so perform_*_fetch
+                            // will not find any new.
+
+                            if let Some(ref watch_folder) = watch_folder {
+                                match self.fetch_new_messages(context, watch_folder).await {
+                                    Ok(res) => {
+                                        info!(context, "fetch_new_messages returned {:?}", res);
+                                        if res {
+                                            break InterruptInfo::new(false, None);
+                                        }
+                                    }
+                                    Err(err) => {
+                                        error!(context, "could not fetch from folder: {}", err);
+                                        self.trigger_reconnect()
+                                    }
+                                }
+                            }
+                        }
+                        Event::Interrupt(info) => {
+                            // Interrupt
+                            break info;
+                        }
+                    }
+                };
+        }
+
+        info!(
+            context,
+            "IMAP-fake-IDLE done after {:.4}s",
+            SystemTime::now()
+                .duration_since(fake_idle_start_time)
+                .unwrap_or_default()
+                .as_millis() as f64
+                / 1000.,
+        );
+
+        info
     }
 }

src/imap/select_folder.rs

@@ -23,17 +23,44 @@ pub enum Error {
 }
 
 impl Imap {
+    /// Issues a CLOSE command to expunge selected folder.
+    ///
+    /// CLOSE is considerably faster than an EXPUNGE, see
+    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
+        if let Some(ref folder) = self.config.selected_folder {
+            info!(context, "Expunge messages in \"{}\".", folder);
+
+            if let Some(ref mut session) = self.session {
+                match session.close().await {
+                    Ok(_) => {
+                        info!(context, "close/expunge succeeded");
+                    }
+                    Err(err) => {
+                        self.trigger_reconnect();
+                        return Err(Error::CloseExpungeFailed(err));
+                    }
+                }
+            } else {
+                return Err(Error::NoSession);
+            }
+        }
+        self.config.selected_folder = None;
+        self.config.selected_folder_needs_expunge = false;
+
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
-        &self,
+        &mut self,
         context: &Context,
         folder: Option<S>,
     ) -> Result<()> {
-        if self.session.lock().await.is_none() {
-            let mut cfg = self.config.write().await;
-            cfg.selected_folder = None;
-            cfg.selected_folder_needs_expunge = false;
+        if self.session.is_none() {
+            self.config.selected_folder = None;
+            self.config.selected_folder_needs_expunge = false;
             self.trigger_reconnect();
             return Err(Error::NoSession);
         }
@@ -41,7 +68,7 @@ impl Imap {
         // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
         // if there is _no_ new folder, we continue as we might want to expunge below.
         if let Some(ref folder) = folder {
-            if let Some(ref selected_folder) = self.config.read().await.selected_folder {
+            if let Some(ref selected_folder) = self.config.selected_folder {
                 if folder.as_ref() == selected_folder {
                     return Ok(());
                 }
@@ -49,33 +76,14 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.read().await.selected_folder_needs_expunge };
+        let needs_expunge = { self.config.selected_folder_needs_expunge };
         if needs_expunge {
-            if let Some(ref folder) = self.config.read().await.selected_folder {
-                info!(context, "Expunge messages in \"{}\".", folder);
-
-                // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see
-                // https://tools.ietf.org/html/rfc3501#section-6.4.2
-                if let Some(ref mut session) = &mut *self.session.lock().await {
-                    match session.close().await {
-                        Ok(_) => {
-                            info!(context, "close/expunge succeeded");
-                        }
-                        Err(err) => {
-                            self.trigger_reconnect();
-                            return Err(Error::CloseExpungeFailed(err));
-                        }
-                    }
-                } else {
-                    return Err(Error::NoSession);
-                }
-            }
-            self.config.write().await.selected_folder_needs_expunge = false;
+            self.close_folder(context).await?;
         }
 
         // select new folder
         if let Some(ref folder) = folder {
-            if let Some(ref mut session) = &mut *self.session.lock().await {
+            if let Some(ref mut session) = &mut self.session {
                 let res = session.select(folder).await;
 
                 // https://tools.ietf.org/html/rfc3501#section-6.3.1
@@ -84,21 +92,20 @@ impl Imap {
 
                 match res {
                     Ok(mailbox) => {
-                        let mut config = self.config.write().await;
-                        config.selected_folder = Some(folder.as_ref().to_string());
-                        config.selected_mailbox = Some(mailbox);
+                        self.config.selected_folder = Some(folder.as_ref().to_string());
+                        self.config.selected_mailbox = Some(mailbox);
                         Ok(())
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
                         self.trigger_reconnect();
-                        self.config.write().await.selected_folder = None;
+                        self.config.selected_folder = None;
                         Err(Error::ConnectionLost)
                     }
                     Err(async_imap::error::Error::Validate(_)) => {
                         Err(Error::BadFolderName(folder.as_ref().to_string()))
                     }
                     Err(err) => {
-                        self.config.write().await.selected_folder = None;
+                        self.config.selected_folder = None;
                         self.trigger_reconnect();
                         Err(Error::Other(err.to_string()))
                     }

src/imap/session.rs

@@ -1,172 +1,40 @@
-use async_imap::{
-    error::Result as ImapResult,
-    types::{Capabilities, Fetch, Mailbox, Name},
-    Session as ImapSession,
-};
+use std::ops::{Deref, DerefMut};
+
+use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
 use async_std::net::TcpStream;
-use async_std::prelude::*;
 
 #[derive(Debug)]
-pub(crate) enum Session {
-    Secure(ImapSession<TlsStream<TcpStream>>),
-    Insecure(ImapSession<TcpStream>),
+pub(crate) struct Session {
+    pub(super) inner: ImapSession<Box<dyn SessionStream>>,
+}
+
+pub(crate) trait SessionStream:
+    async_std::io::Read + async_std::io::Write + Unpin + Send + Sync + std::fmt::Debug
+{
+}
+
+impl SessionStream for TlsStream<Box<dyn SessionStream>> {}
+impl SessionStream for TlsStream<TcpStream> {}
+impl SessionStream for TcpStream {}
+
+impl Deref for Session {
+    type Target = ImapSession<Box<dyn SessionStream>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl DerefMut for Session {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
+    }
 }
 
 impl Session {
-    pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
-        let res = match self {
-            Session::Secure(i) => i.capabilities().await?,
-            Session::Insecure(i) => i.capabilities().await?,
-        };
-
-        Ok(res)
-    }
-
-    pub async fn list(
-        &mut self,
-        reference_name: Option<&str>,
-        mailbox_pattern: Option<&str>,
-    ) -> ImapResult<Vec<Name>> {
-        let res = match self {
-            Session::Secure(i) => {
-                i.list(reference_name, mailbox_pattern)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.list(reference_name, mailbox_pattern)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-        Ok(res)
-    }
-
-    pub async fn create<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.create(mailbox_name).await?,
-            Session::Insecure(i) => i.create(mailbox_name).await?,
-        }
-        Ok(())
-    }
-
-    pub async fn subscribe<S: AsRef<str>>(&mut self, mailbox: S) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.subscribe(mailbox).await?,
-            Session::Insecure(i) => i.subscribe(mailbox).await?,
-        }
-        Ok(())
-    }
-
-    pub async fn close(&mut self) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.close().await?,
-            Session::Insecure(i) => i.close().await?,
-        }
-        Ok(())
-    }
-
-    pub async fn select<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<Mailbox> {
-        let mbox = match self {
-            Session::Secure(i) => i.select(mailbox_name).await?,
-            Session::Insecure(i) => i.select(mailbox_name).await?,
-        };
-
-        Ok(mbox)
-    }
-
-    pub async fn fetch<S1, S2>(&mut self, sequence_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
-    where
-        S1: AsRef<str>,
-        S2: AsRef<str>,
-    {
-        let res = match self {
-            Session::Secure(i) => {
-                i.fetch(sequence_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.fetch(sequence_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-        Ok(res)
-    }
-
-    pub async fn uid_fetch<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
-    where
-        S1: AsRef<str>,
-        S2: AsRef<str>,
-    {
-        let res = match self {
-            Session::Secure(i) => {
-                i.uid_fetch(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.uid_fetch(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-
-        Ok(res)
-    }
-
-    pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
-    where
-        S1: AsRef<str>,
-        S2: AsRef<str>,
-    {
-        let res = match self {
-            Session::Secure(i) => {
-                i.uid_store(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-            Session::Insecure(i) => {
-                i.uid_store(uid_set, query)
-                    .await?
-                    .collect::<ImapResult<_>>()
-                    .await?
-            }
-        };
-        Ok(res)
-    }
-
-    pub async fn uid_mv<S1: AsRef<str>, S2: AsRef<str>>(
-        &mut self,
-        uid_set: S1,
-        mailbox_name: S2,
-    ) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.uid_mv(uid_set, mailbox_name).await?,
-            Session::Insecure(i) => i.uid_mv(uid_set, mailbox_name).await?,
-        }
-        Ok(())
-    }
-
-    pub async fn uid_copy<S1: AsRef<str>, S2: AsRef<str>>(
-        &mut self,
-        uid_set: S1,
-        mailbox_name: S2,
-    ) -> ImapResult<()> {
-        match self {
-            Session::Secure(i) => i.uid_copy(uid_set, mailbox_name).await?,
-            Session::Insecure(i) => i.uid_copy(uid_set, mailbox_name).await?,
-        }
-
-        Ok(())
+    pub fn idle(self) -> async_imap::extensions::idle::Handle<Box<dyn SessionStream>> {
+        let Session { inner } = self;
+        inner.idle()
     }
 }

src/imex.rs

@@ -1,9 +1,10 @@
 //! # Import/export module
 
-use core::cmp::{max, min};
-use std::path::Path;
+use std::any::Any;
+use std::cmp::{max, min};
 
-use num_traits::FromPrimitive;
+use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -16,8 +17,7 @@ use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
-use crate::job::*;
-use crate::key::{self, DcKey, Key, SignedSecretKey};
+use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -53,13 +53,10 @@ pub enum ImexMode {
 }
 
 /// Import/export things.
-/// For this purpose, the function creates a job that is executed in the IMAP-thread then;
-/// this requires to call dc_perform_inbox_jobs() regularly.
 ///
 /// What to do is defined by the *what* parameter.
 ///
-/// While dc_imex() returns immediately, the started job may take a while,
-/// you can stop it using dc_stop_ongoing_process(). During execution of the job,
+/// During execution of the job,
 /// some events are sent out:
 ///
 /// - A number of #DC_EVENT_IMEX_PROGRESS events are sent and may be used to create
@@ -68,73 +65,84 @@ pub enum ImexMode {
 /// - For each file written on export, the function sends #DC_EVENT_IMEX_FILE_WRITTEN
 ///
 /// Only one import-/export-progress can run at the same time.
-/// To cancel an import-/export-progress, use dc_stop_ongoing_process().
-pub fn imex(context: &Context, what: ImexMode, param1: Option<impl AsRef<Path>>) {
-    let mut param = Params::new();
-    param.set_int(Param::Cmd, what as i32);
-    if let Some(param1) = param1 {
-        param.set(Param::Arg, param1.as_ref().to_string_lossy());
-    }
+/// To cancel an import-/export-progress, drop the future returned by this function.
+pub async fn imex(
+    context: &Context,
+    what: ImexMode,
+    param1: Option<impl AsRef<Path>>,
+) -> Result<()> {
+    use futures::future::FutureExt;
 
-    job_kill_action(context, Action::ImexImap);
-    job_add(context, Action::ImexImap, 0, param, 0);
+    let cancel = context.alloc_ongoing().await?;
+    let res = imex_inner(context, what, param1)
+        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
+        .await;
+
+    context.free_ongoing().await;
+
+    res
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
-    let dir_iter = std::fs::read_dir(dir_name)?;
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
-    let mut newest_backup_path: Option<std::path::PathBuf> = None;
-    for dirent in dir_iter {
+    let mut newest_backup_path: Option<PathBuf> = None;
+    while let Some(dirent) = dir_iter.next().await {
         if let Ok(dirent) = dirent {
             let path = dirent.path();
             let name = dirent.file_name();
             let name = name.to_string_lossy();
             if name.starts_with("delta-chat") && name.ends_with(".bak") {
                 let sql = Sql::new();
-                if sql.open(context, &path, true) {
-                    let curr_backup_time = sql
-                        .get_raw_config_int(context, "backup_time")
-                        .unwrap_or_default();
-                    if curr_backup_time > newest_backup_time {
-                        newest_backup_path = Some(path);
-                        newest_backup_time = curr_backup_time;
-                    }
-                    info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                    sql.close(&context);
+                sql.open(context, &path, true).await?;
+                let curr_backup_time = sql
+                    .get_raw_config_int("backup_time")
+                    .await
+                    .unwrap_or_default();
+                if curr_backup_time > newest_backup_time {
+                    newest_backup_path = Some(path);
+                    newest_backup_time = curr_backup_time;
                 }
+                info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                sql.close().await;
             }
         }
     }
+
     match newest_backup_path {
         Some(path) => Ok(path.to_string_lossy().into_owned()),
         None => bail!("no backup found in {}", dir_name.display()),
     }
 }
 
-pub fn initiate_key_transfer(context: &Context) -> Result<String> {
-    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
-    let res = do_initiate_key_transfer(context);
-    context.free_ongoing();
+pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
+    use futures::future::FutureExt;
+
+    let cancel = context.alloc_ongoing().await?;
+    let res = do_initiate_key_transfer(context)
+        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
+        .await;
+
+    context.free_ongoing().await;
     res
 }
 
-fn do_initiate_key_transfer(context: &Context) -> Result<String> {
+async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     let mut msg: Message;
     let setup_code = create_setup_code(context);
     /* this may require a keypair to be created. this may take a second ... */
-    ensure!(!context.shall_stop_ongoing(), "canceled");
-    let setup_file_content = render_setup_file(context, &setup_code)?;
+    let setup_file_content = render_setup_file(context, &setup_code).await?;
     /* encrypting may also take a while ... */
-    ensure!(!context.shall_stop_ongoing(), "canceled");
     let setup_file_blob = BlobObject::create(
         context,
         "autocrypt-setup-message.html",
         setup_file_content.as_bytes(),
-    )?;
+    )
+    .await?;
 
-    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF)?;
+    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF).await?;
     msg = Message::default();
     msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
@@ -147,12 +155,11 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
         ForcePlaintext::NoAutocryptHeader as i32,
     );
 
-    ensure!(!context.shall_stop_ongoing(), "canceled");
-    let msg_id = chat::send_msg(context, chat_id, &mut msg)?;
+    let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
-    while !context.shall_stop_ongoing() {
-        std::thread::sleep(std::time::Duration::from_secs(1));
-        if let Ok(msg) = Message::load_from_db(context, msg_id) {
+    while !context.shall_stop_ongoing().await {
+        async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+        if let Ok(msg) = Message::load_from_db(context, msg_id).await {
             if msg.is_sent() {
                 info!(context, "... setup message sent.",);
                 break;
@@ -170,18 +177,18 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 /// Renders HTML body of a setup file message.
 ///
 /// The `passphrase` must be at least 2 characters long.
-pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
+pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
     ensure!(
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let private_key = Key::from(SignedSecretKey::load_self(context)?);
-    let ac_headers = match context.get_config_bool(Config::E2eeEnabled) {
+    let private_key = SignedSecretKey::load_self(context).await?;
+    let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
+    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -193,8 +200,8 @@ pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String>
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
-    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject);
-    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody);
+    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject).await;
+    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody).await;
     let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
     Ok(format!(
         concat!(
@@ -237,8 +244,8 @@ pub fn create_setup_code(_context: &Context) -> String {
     ret
 }
 
-fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
-    if !context.sql.get_raw_config_bool(context, "bcc_self") {
+async fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
+    if !context.sql.get_raw_config_bool("bcc_self").await {
         let mut msg = Message::new(Viewtype::Text);
         // TODO: define this as a stockstring once the wording is settled.
         msg.text = Some(
@@ -247,26 +254,30 @@ fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
              go to the settings and enable \"Send copy to self\"."
                 .to_string(),
         );
-        chat::add_device_msg(context, Some("bcc-self-hint"), Some(&mut msg))?;
+        chat::add_device_msg(context, Some("bcc-self-hint"), Some(&mut msg)).await?;
     }
     Ok(())
 }
 
-pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str) -> Result<()> {
+pub async fn continue_key_transfer(
+    context: &Context,
+    msg_id: MsgId,
+    setup_code: &str,
+) -> Result<()> {
     ensure!(!msg_id.is_special(), "wrong id");
 
-    let msg = Message::load_from_db(context, msg_id)?;
+    let msg = Message::load_from_db(context, msg_id).await?;
     ensure!(
         msg.is_setupmessage(),
         "Message is no Autocrypt Setup Message."
     );
 
     if let Some(filename) = msg.get_file(context) {
-        let file = dc_open_file(context, filename)?;
+        let file = dc_open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
-        let armored_key = decrypt_setup_file(context, &sc, file)?;
-        set_self_key(context, &armored_key, true, true)?;
-        maybe_add_bcc_self_device_msg(context)?;
+        let armored_key = decrypt_setup_file(&sc, file).await?;
+        set_self_key(context, &armored_key, true, true).await?;
+        maybe_add_bcc_self_device_msg(context).await?;
 
         Ok(())
     } else {
@@ -274,19 +285,15 @@ pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str)
     }
 }
 
-fn set_self_key(
+async fn set_self_key(
     context: &Context,
     armored: &str,
     set_default: bool,
     prefer_encrypt_required: bool,
 ) -> Result<()> {
     // try hard to only modify key-state
-    let keys = Key::from_armored_string(armored, KeyType::Private)
-        .and_then(|(k, h)| if k.verify() { Some((k, h)) } else { None })
-        .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
-
-    ensure!(keys.is_some(), "Not a valid private key");
-    let (private_key, public_key, header) = keys.unwrap();
+    let (private_key, header) = SignedSecretKey::from_asc(armored)?;
+    let public_key = private_key.split_public_key()?;
     let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
     match preferencrypt.map(|s| s.as_str()) {
         Some(headerval) => {
@@ -299,7 +306,8 @@ fn set_self_key(
             };
             context
                 .sql
-                .set_raw_config_int(context, "e2ee_enabled", e2ee_enabled)?;
+                .set_raw_config_int(context, "e2ee_enabled", e2ee_enabled)
+                .await?;
         }
         None => {
             if prefer_encrypt_required {
@@ -308,18 +316,13 @@ fn set_self_key(
         }
     };
 
-    let self_addr = context.get_config(Config::ConfiguredAddr);
+    let self_addr = context.get_config(Config::ConfiguredAddr).await;
     ensure!(self_addr.is_some(), "Missing self addr");
     let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
-
-    let (public, secret) = match (public_key, private_key) {
-        (Key::Public(p), Key::Secret(s)) => (p, s),
-        _ => bail!("wrong keys unpacked"),
-    };
     let keypair = pgp::KeyPair {
         addr,
-        public,
-        secret,
+        public: public_key,
+        secret: private_key,
     };
     key::store_self_keypair(
         context,
@@ -329,16 +332,16 @@ fn set_self_key(
         } else {
             key::KeyPairUse::ReadOnly
         },
-    )?;
+    )
+    .await?;
     Ok(())
 }
 
-fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
-    _context: &Context,
+async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     file: T,
 ) -> Result<String> {
-    let plain_bytes = pgp::symm_decrypt(passphrase, file)?;
+    let plain_bytes = pgp::symm_decrypt(passphrase, file).await?;
     let plain_text = std::string::String::from_utf8(plain_bytes)?;
 
     Ok(plain_text)
@@ -357,52 +360,51 @@ pub fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-#[allow(non_snake_case)]
-pub fn JobImexImap(context: &Context, job: &Job) -> Result<()> {
-    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
-    let what: Option<ImexMode> = job.param.get_int(Param::Cmd).and_then(ImexMode::from_i32);
-    let param = job.param.get(Param::Arg).unwrap_or_default();
+async fn imex_inner(
+    context: &Context,
+    what: ImexMode,
+    param: Option<impl AsRef<Path>>,
+) -> Result<()> {
+    ensure!(param.is_some(), "No Import/export dir/file given.");
 
-    ensure!(!param.is_empty(), "No Import/export dir/file given.");
     info!(context, "Import/export process started.");
-    context.call_cb(Event::ImexProgress(10));
+    context.emit_event(Event::ImexProgress(10));
 
-    ensure!(context.sql.is_open(), "Database not opened.");
-    if what == Some(ImexMode::ExportBackup) || what == Some(ImexMode::ExportSelfKeys) {
+    ensure!(context.sql.is_open().await, "Database not opened.");
+
+    let path = param.unwrap();
+    if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
-        if e2ee::ensure_secret_key_exists(context).is_err() {
-            context.free_ongoing();
+        if e2ee::ensure_secret_key_exists(context).await.is_err() {
             bail!("Cannot create private key or private key not available.");
         } else {
-            dc_create_folder(context, &param)?;
+            dc_create_folder(context, &path).await?;
         }
     }
-    let path = Path::new(param);
+
     let success = match what {
-        Some(ImexMode::ExportSelfKeys) => export_self_keys(context, path),
-        Some(ImexMode::ImportSelfKeys) => import_self_keys(context, path),
-        Some(ImexMode::ExportBackup) => export_backup(context, path),
-        Some(ImexMode::ImportBackup) => import_backup(context, path),
-        None => {
-            bail!("unknown IMEX type");
-        }
+        ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
+        ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
+        ImexMode::ExportBackup => export_backup(context, path).await,
+        ImexMode::ImportBackup => import_backup(context, path).await,
     };
-    context.free_ongoing();
+
     match success {
         Ok(()) => {
             info!(context, "IMEX successfully completed");
-            context.call_cb(Event::ImexProgress(1000));
+            context.emit_event(Event::ImexProgress(1000));
             Ok(())
         }
         Err(err) => {
-            context.call_cb(Event::ImexProgress(0));
+            error!(context, "IMEX FAILED: {}", err);
+            context.emit_event(Event::ImexProgress(0));
             bail!("IMEX FAILED to complete: {}", err);
         }
     }
 }
 
 /// Import Backup
-fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -411,85 +413,81 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     );
 
     ensure!(
-        !context.is_configured(),
+        !context.is_configured().await,
         "Cannot import backups to accounts in use."
     );
-    context.sql.close(&context);
-    dc_delete_file(context, context.get_dbfile());
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
     ensure!(
-        !context.get_dbfile().exists(),
+        !context.get_dbfile().exists().await,
         "Cannot delete old database."
     );
 
     ensure!(
-        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()),
+        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()).await,
         "could not copy file"
     );
     /* error already logged */
     /* re-open copied database file */
-    ensure!(
-        context.sql.open(&context, &context.get_dbfile(), false),
-        "could not re-open db"
-    );
-
-    delete_and_reset_all_device_msgs(&context)?;
-
-    let total_files_cnt = context
+    context
         .sql
-        .query_get_value::<_, isize>(context, "SELECT COUNT(*) FROM backup_blobs;", params![])
-        .unwrap_or_default() as usize;
+        .open(&context, &context.get_dbfile(), false)
+        .await?;
+
+    delete_and_reset_all_device_msgs(&context).await?;
+
+    let total_files_cnt: i32 = context
+        .sql
+        .query_value("SELECT COUNT(*) FROM backup_blobs;", paramsx![])
+        .await
+        .unwrap_or_default();
+    let total_files_cnt = total_files_cnt as usize;
     info!(
         context,
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
     );
 
-    let res = context.sql.query_map(
-        "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
-        params![],
-        |row| {
-            let name: String = row.get(0)?;
-            let blob: Vec<u8> = row.get(1)?;
+    let pool = context.sql.get_pool().await?;
+    let mut files = sqlx::query_as("SELECT file_name, file_content FROM backup_blobs ORDER BY id;")
+        .fetch(&pool);
 
-            Ok((name, blob))
-        },
-        |files| {
-            for (processed_files_cnt, file) in files.enumerate() {
-                let (file_name, file_blob) = file?;
-                if context.shall_stop_ongoing() {
-                    return Ok(false);
-                }
-                let mut permille = processed_files_cnt * 1000 / total_files_cnt;
-                if permille < 10 {
-                    permille = 10
-                }
-                if permille > 990 {
-                    permille = 990
-                }
-                context.call_cb(Event::ImexProgress(permille));
-                if file_blob.is_empty() {
-                    continue;
-                }
+    let mut processed_files_cnt = 0;
+    while let Some(files_result) = files.next().await {
+        let (file_name, file_blob): (String, Vec<u8>) = files_result?;
+        if context.shall_stop_ongoing().await {
+            break;
+        }
 
-                let path_filename = context.get_blobdir().join(file_name);
-                dc_write_file(context, &path_filename, &file_blob)?;
-            }
-            Ok(true)
-        },
+        let mut permille = processed_files_cnt * 1000 / total_files_cnt;
+        if permille < 10 {
+            permille = 10
+        }
+        if permille > 990 {
+            permille = 990
+        }
+        context.emit_event(Event::ImexProgress(permille));
+        if file_blob.is_empty() {
+            continue;
+        }
+
+        let path_filename = context.get_blobdir().join(file_name);
+        dc_write_file(context, &path_filename, &file_blob).await?;
+
+        processed_files_cnt += 1;
+    }
+
+    ensure!(
+        processed_files_cnt == total_files_cnt,
+        "received stop signal"
     );
 
-    match res {
-        Ok(all_files_extracted) => {
-            if all_files_extracted {
-                // only delete backup_blobs if all files were successfully extracted
-                sql::execute(context, &context.sql, "DROP TABLE backup_blobs;", params![])?;
-                sql::try_execute(context, &context.sql, "VACUUM;").ok();
-                Ok(())
-            } else {
-                bail!("received stop signal");
-            }
-        }
-        Err(err) => Err(err.into()),
-    }
+    context
+        .sql
+        .execute("DROP TABLE backup_blobs;", paramsx![])
+        .await?;
+    context.sql.execute("VACUUM;", paramsx![]).await?;
+
+    Ok(())
 }
 
 /*******************************************************************************
@@ -497,28 +495,31 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
  ******************************************************************************/
 /* the FILE_PROGRESS macro calls the callback with the permille of files processed.
 The macro avoids weird values of 0% or 100% while still working. */
-fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let dest_path_filename = dc_get_next_backup_path(dir, now)?;
+    let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
     let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
-    sql::housekeeping(context);
+    sql::housekeeping(context).await?;
 
-    sql::try_execute(context, &context.sql, "VACUUM;").ok();
+    context.sql.execute("VACUUM;", paramsx![]).await.ok();
 
     // we close the database during the copy of the dbfile
-    context.sql.close(context);
+    context.sql.close().await;
     info!(
         context,
         "Backup '{}' to '{}'.",
         context.get_dbfile().display(),
         dest_path_filename.display(),
     );
-    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
-    context.sql.open(&context, &context.get_dbfile(), false);
+    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename).await;
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await?;
 
     if !copied {
         bail!(
@@ -528,87 +529,85 @@ fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         );
     }
     let dest_sql = Sql::new();
-    ensure!(
-        dest_sql.open(context, &dest_path_filename, false),
-        "could not open exported database {}",
-        dest_path_string
-    );
-    let res = match add_files_to_export(context, &dest_sql) {
+    dest_sql.open(context, &dest_path_filename, false).await?;
+
+    let res = match add_files_to_export(context, &dest_sql).await {
+        Ok(()) => {
+            dest_sql
+                .set_raw_config_int(context, "backup_time", now as i32)
+                .await?;
+            context.emit_event(Event::ImexFileWritten(dest_path_filename));
+            Ok(())
+        }
         Err(err) => {
-            dc_delete_file(context, &dest_path_filename);
+            dc_delete_file(context, &dest_path_filename).await;
             error!(context, "backup failed: {}", err);
             Err(err)
         }
-        Ok(()) => {
-            dest_sql.set_raw_config_int(context, "backup_time", now as i32)?;
-            context.call_cb(Event::ImexFileWritten(dest_path_filename));
-            Ok(())
-        }
     };
-    dest_sql.close(context);
+    dest_sql.close().await;
 
     Ok(res?)
 }
 
-fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
+async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
     // add all files as blobs to the database copy (this does not require
     // the source to be locked, neigher the destination as it is used only here)
-    if !sql.table_exists("backup_blobs") {
-        sql::execute(
-            context,
-            &sql,
+    if !sql.table_exists("backup_blobs").await? {
+        sql.execute(
             "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-            params![],
-        )?
+            paramsx![],
+        )
+        .await?;
     }
     // copy all files from BLOBDIR into backup-db
     let mut total_files_cnt = 0;
     let dir = context.get_blobdir();
-    let dir_handle = std::fs::read_dir(&dir)?;
-    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
+    let dir_handle = async_std::fs::read_dir(&dir).await?;
+    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count().await;
 
     info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-    // scan directory, pass 2: copy files
-    let dir_handle = std::fs::read_dir(&dir)?;
-    let exported_all_files = sql.prepare(
-        "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
-        |mut stmt, _| {
-            let mut processed_files_cnt = 0;
-            for entry in dir_handle {
-                let entry = entry?;
-                if context.shall_stop_ongoing() {
-                    return Ok(false);
-                }
-                processed_files_cnt += 1;
-                let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-                context.call_cb(Event::ImexProgress(permille));
 
-                let name_f = entry.file_name();
-                let name = name_f.to_string_lossy();
-                if name.starts_with("delta-chat") && name.ends_with(".bak") {
-                    continue;
-                }
-                info!(context, "EXPORT: copying filename={}", name);
-                let curr_path_filename = context.get_blobdir().join(entry.file_name());
-                if let Ok(buf) = dc_read_file(context, &curr_path_filename) {
-                    if buf.is_empty() {
-                        continue;
-                    }
-                    // bail out if we can't insert
-                    stmt.execute(params![name, buf])?;
-                }
+    // scan directory, pass 2: copy files
+    let mut dir_handle = async_std::fs::read_dir(&dir).await?;
+
+    let mut processed_files_cnt = 0;
+    while let Some(entry) = dir_handle.next().await {
+        let entry = entry?;
+        if context.shall_stop_ongoing().await {
+            return Ok(());
+        }
+        processed_files_cnt += 1;
+        let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
+        context.emit_event(Event::ImexProgress(permille));
+
+        let name_f = entry.file_name();
+        let name = name_f.to_string_lossy();
+        if name.starts_with("delta-chat") && name.ends_with(".bak") {
+            continue;
+        }
+        info!(context, "EXPORT: copying filename={}", name);
+        let curr_path_filename = context.get_blobdir().join(entry.file_name());
+        if let Ok(buf) = dc_read_file(context, &curr_path_filename).await {
+            if buf.is_empty() {
+                continue;
             }
-            Ok(true)
-        },
-    )?;
-    ensure!(exported_all_files, "canceled during export-files");
+            // bail out if we can't insert
+            sql.execute(
+                "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
+                paramsx![name.as_ref(), buf],
+            )
+            .await?;
+        }
+    }
+
     Ok(())
 }
 
 /*******************************************************************************
  * Classic key import
  ******************************************************************************/
-fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
     plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
     Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
@@ -619,8 +618,8 @@ fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     let mut imported_cnt = 0;
 
     let dir_name = dir.as_ref().to_string_lossy();
-    let dir_handle = std::fs::read_dir(&dir)?;
-    for entry in dir_handle {
+    let mut dir_handle = async_std::fs::read_dir(&dir).await?;
+    while let Some(entry) = dir_handle.next().await {
         let entry_fn = entry?.file_name();
         let name_f = entry_fn.to_string_lossy();
         let path_plus_name = dir.as_ref().join(&entry_fn);
@@ -640,10 +639,10 @@ fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
                 continue;
             }
         }
-        match dc_read_file(context, &path_plus_name) {
+        match dc_read_file(context, &path_plus_name).await {
             Ok(buf) => {
                 let armored = std::string::String::from_utf8_lossy(&buf);
-                if let Err(err) = set_self_key(context, &armored, set_default, false) {
+                if let Err(err) = set_self_key(context, &armored, set_default, false).await {
                     error!(context, "set_self_key: {}", err);
                     continue;
                 }
@@ -660,73 +659,82 @@ fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     Ok(())
 }
 
-fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     let mut export_errors = 0;
 
-    context.sql.query_map(
-        "SELECT id, public_key, private_key, is_default FROM keypairs;",
-        params![],
-        |row| {
-            let id = row.get(0)?;
-            let public_key_blob: Vec<u8> = row.get(1)?;
-            let public_key = Key::from_slice(&public_key_blob, KeyType::Public);
-            let private_key_blob: Vec<u8> = row.get(2)?;
-            let private_key = Key::from_slice(&private_key_blob, KeyType::Private);
-            let is_default: i32 = row.get(3)?;
+    let pool = context.sql.get_pool().await?;
 
-            Ok((id, public_key, private_key, is_default))
-        },
-        |keys| {
-            for key_pair in keys {
-                let (id, public_key, private_key, is_default) = key_pair?;
-                let id = Some(id).filter(|_| is_default != 0);
-                if let Some(key) = public_key {
-                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
-                        export_errors += 1;
-                    }
-                } else {
-                    export_errors += 1;
-                }
-                if let Some(key) = private_key {
-                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
-                        export_errors += 1;
-                    }
-                } else {
-                    export_errors += 1;
-                }
+    let mut keys = sqlx::query_as("SELECT id, public_key, private_key, is_default FROM keypairs;")
+        .fetch(&pool);
+
+    while let Some(keys_result) = keys.next().await {
+        let (id, public_key_blob, private_key_blob, is_default): (i64, Vec<u8>, Vec<u8>, i32) =
+            keys_result?;
+        let public_key = SignedPublicKey::from_slice(&public_key_blob);
+        let private_key = SignedSecretKey::from_slice(&private_key_blob);
+
+        let id = Some(id).filter(|_| is_default != 0);
+
+        if let Ok(key) = public_key {
+            if export_key_to_asc_file(context, &dir, id, &key)
+                .await
+                .is_err()
+            {
+                export_errors += 1;
             }
-
-            Ok(())
-        },
-    )?;
+        } else {
+            export_errors += 1;
+        }
+        if let Ok(key) = private_key {
+            if export_key_to_asc_file(context, &dir, id, &key)
+                .await
+                .is_err()
+            {
+                export_errors += 1;
+            }
+        } else {
+            export_errors += 1;
+        }
+    }
 
     ensure!(export_errors == 0, "errors while exporting keys");
+
     Ok(())
 }
 
 /*******************************************************************************
  * Classic key export
  ******************************************************************************/
-fn export_key_to_asc_file(
+async fn export_key_to_asc_file<T>(
     context: &Context,
     dir: impl AsRef<Path>,
     id: Option<i64>,
-    key: &Key,
-) -> std::io::Result<()> {
+    key: &T,
+) -> std::io::Result<()>
+where
+    T: DcKey + Any,
+{
     let file_name = {
-        let kind = if key.is_public() { "public" } else { "private" };
+        let any_key = key as &dyn Any;
+        let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "public"
+        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "private"
+        } else {
+            "unknown"
+        };
         let id = id.map_or("default".into(), |i| i.to_string());
-
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
     info!(context, "Exporting key {}", file_name.display());
-    dc_delete_file(context, &file_name);
+    dc_delete_file(context, &file_name).await;
 
-    let res = key.write_asc_to_file(&file_name, context);
+    let content = key.to_asc(None).into_bytes();
+    let res = dc_write_file(context, &file_name, &content).await;
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
-        context.call_cb(Event::ImexFileWritten(file_name));
+        context.emit_event(Event::ImexFileWritten(file_name));
     }
     res
 }
@@ -738,12 +746,12 @@ mod tests {
     use crate::test_utils::*;
     use ::pgp::armor::BlockType;
 
-    #[test]
-    fn test_render_setup_file() {
-        let t = test_context(Some(Box::new(logging_cb)));
+    #[async_std::test]
+    async fn test_render_setup_file() {
+        let t = TestContext::new().await;
 
-        configure_alice_keypair(&t.ctx);
-        let msg = render_setup_file(&t.ctx, "hello").unwrap();
+        t.configure_alice().await;
+        let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
         // In particular note the mixing of `\r\n` and `\n` depending
@@ -757,21 +765,22 @@ mod tests {
         assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 
-    #[test]
-    fn test_render_setup_file_newline_replace() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_render_setup_file_newline_replace() {
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+            .await
             .unwrap();
-        configure_alice_keypair(&t.ctx);
-        let msg = render_setup_file(&t.ctx, "pw").unwrap();
+        t.configure_alice().await;
+        let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
     }
 
-    #[test]
-    fn test_create_setup_code() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_create_setup_code() {
+        let t = TestContext::new().await;
         let setupcode = create_setup_code(&t.ctx);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
@@ -784,15 +793,17 @@ mod tests {
         assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
     }
 
-    #[test]
-    fn test_export_key_to_asc_file() {
-        let context = dummy_context();
-        let key = Key::from(alice_keypair().public);
+    #[async_std::test]
+    async fn test_export_key_to_asc_file() {
+        let context = TestContext::new().await;
+        let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
-        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key).is_ok());
+        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
+            .await
+            .is_ok());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{}/public-key-default.asc", blobdir);
-        let bytes = std::fs::read(&filename).unwrap();
+        let bytes = async_std::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
@@ -812,11 +823,8 @@ mod tests {
     const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
     const S_EM_SETUPFILE: &str = include_str!("../test-data/message/stress.txt");
 
-    #[test]
-    fn test_split_and_decrypt() {
-        let ctx = dummy_context();
-        let context = &ctx.ctx;
-
+    #[async_std::test]
+    async fn test_split_and_decrypt() {
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
         let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
         assert_eq!(typ, BlockType::Message);
@@ -826,12 +834,10 @@ mod tests {
         assert!(!base64.is_empty());
 
         let setup_file = S_EM_SETUPFILE.to_string();
-        let decrypted = decrypt_setup_file(
-            context,
-            S_EM_SETUPCODE,
-            std::io::Cursor::new(setup_file.as_bytes()),
-        )
-        .unwrap();
+        let decrypted =
+            decrypt_setup_file(S_EM_SETUPCODE, std::io::Cursor::new(setup_file.as_bytes()))
+                .await
+                .unwrap();
 
         let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
 

src/job_thread.rs

@@ -1,203 +0,0 @@
-use std::sync::{Arc, Condvar, Mutex};
-
-use crate::context::Context;
-use crate::error::{format_err, Result};
-use crate::imap::Imap;
-
-#[derive(Debug)]
-pub struct JobThread {
-    pub name: &'static str,
-    pub folder_config_name: &'static str,
-    pub imap: Imap,
-    pub state: Arc<(Mutex<JobState>, Condvar)>,
-}
-
-#[derive(Clone, Debug, Default)]
-pub struct JobState {
-    idle: bool,
-    jobs_needed: bool,
-    suspended: bool,
-    using_handle: bool,
-}
-
-impl JobThread {
-    pub fn new(name: &'static str, folder_config_name: &'static str, imap: Imap) -> Self {
-        JobThread {
-            name,
-            folder_config_name,
-            imap,
-            state: Arc::new((Mutex::new(Default::default()), Condvar::new())),
-        }
-    }
-
-    pub fn suspend(&self, context: &Context) {
-        info!(context, "Suspending {}-thread.", self.name,);
-        {
-            self.state.0.lock().unwrap().suspended = true;
-        }
-        self.interrupt_idle(context);
-        loop {
-            let using_handle = self.state.0.lock().unwrap().using_handle;
-            if !using_handle {
-                return;
-            }
-            std::thread::sleep(std::time::Duration::from_micros(300 * 1000));
-        }
-    }
-
-    pub fn unsuspend(&self, context: &Context) {
-        info!(context, "Unsuspending {}-thread.", self.name);
-
-        let &(ref lock, ref cvar) = &*self.state.clone();
-        let mut state = lock.lock().unwrap();
-
-        state.suspended = false;
-        state.idle = true;
-        cvar.notify_one();
-    }
-
-    pub fn interrupt_idle(&self, context: &Context) {
-        {
-            self.state.0.lock().unwrap().jobs_needed = true;
-        }
-
-        info!(context, "Interrupting {}-IDLE...", self.name);
-
-        self.imap.interrupt_idle(context);
-
-        let &(ref lock, ref cvar) = &*self.state.clone();
-        let mut state = lock.lock().unwrap();
-
-        state.idle = true;
-        cvar.notify_one();
-        info!(context, "Interrupting {}-IDLE... finished", self.name);
-    }
-
-    pub async fn fetch(&mut self, context: &Context, use_network: bool) {
-        {
-            let &(ref lock, _) = &*self.state.clone();
-            let mut state = lock.lock().unwrap();
-
-            if state.suspended {
-                return;
-            }
-
-            state.using_handle = true;
-        }
-
-        if use_network {
-            if let Err(err) = self.connect_and_fetch(context).await {
-                warn!(context, "connect+fetch failed: {}, reconnect & retry", err);
-                self.imap.trigger_reconnect();
-                if let Err(err) = self.connect_and_fetch(context).await {
-                    warn!(context, "connect+fetch failed: {}", err);
-                }
-            }
-        }
-        self.state.0.lock().unwrap().using_handle = false;
-    }
-
-    async fn connect_and_fetch(&mut self, context: &Context) -> Result<()> {
-        let prefix = format!("{}-fetch", self.name);
-        self.imap.connect_configured(context)?;
-        if let Some(watch_folder) = self.get_watch_folder(context) {
-            let start = std::time::Instant::now();
-            info!(context, "{} started...", prefix);
-            let res = self
-                .imap
-                .fetch(context, &watch_folder)
-                .await
-                .map_err(Into::into);
-            let elapsed = start.elapsed().as_millis();
-            info!(context, "{} done in {:.3} ms.", prefix, elapsed);
-
-            res
-        } else {
-            Err(format_err!("WatchFolder not found: not-set"))
-        }
-    }
-
-    fn get_watch_folder(&self, context: &Context) -> Option<String> {
-        match context.sql.get_raw_config(context, self.folder_config_name) {
-            Some(name) => Some(name),
-            None => {
-                if self.folder_config_name == "configured_inbox_folder" {
-                    // initialized with old version, so has not set configured_inbox_folder
-                    Some("INBOX".to_string())
-                } else {
-                    None
-                }
-            }
-        }
-    }
-
-    pub fn idle(&self, context: &Context, use_network: bool) {
-        {
-            let &(ref lock, ref cvar) = &*self.state.clone();
-            let mut state = lock.lock().unwrap();
-
-            if state.jobs_needed {
-                info!(
-                    context,
-                    "{}-IDLE will not be started as it was interrupted while not idling.",
-                    self.name,
-                );
-                state.jobs_needed = false;
-                return;
-            }
-
-            if state.suspended {
-                while !state.idle {
-                    state = cvar.wait(state).unwrap();
-                }
-                state.idle = false;
-                return;
-            }
-
-            state.using_handle = true;
-
-            if !use_network {
-                state.using_handle = false;
-
-                while !state.idle {
-                    state = cvar.wait(state).unwrap();
-                }
-                state.idle = false;
-                return;
-            }
-        }
-
-        let prefix = format!("{}-IDLE", self.name);
-        let do_fake_idle = match self.imap.connect_configured(context) {
-            Ok(()) => {
-                if !self.imap.can_idle() {
-                    true // we have to do fake_idle
-                } else {
-                    let watch_folder = self.get_watch_folder(context);
-                    info!(context, "{} started...", prefix);
-                    let res = self.imap.idle(context, watch_folder);
-                    info!(context, "{} ended...", prefix);
-                    if let Err(err) = res {
-                        warn!(context, "{} failed: {} -> reconnecting", prefix, err);
-                        // something is borked, let's start afresh on the next occassion
-                        self.imap.disconnect(context);
-                    }
-                    false
-                }
-            }
-            Err(err) => {
-                info!(context, "{}-IDLE connection fail: {:?}", self.name, err);
-                // if the connection fails, use fake_idle to retry periodically
-                // fake_idle() will be woken up by interrupt_idle() as
-                // well so will act on maybe_network events
-                true
-            }
-        };
-        if do_fake_idle {
-            let watch_folder = self.get_watch_folder(context);
-            self.imap.fake_idle(context, watch_folder);
-        }
-
-        self.state.0.lock().unwrap().using_handle = false;
-    }
-}

src/key.rs

@@ -1,18 +1,20 @@
 //! Cryptographic key module
 
 use std::collections::BTreeMap;
+use std::fmt;
 use std::io::Cursor;
-use std::path::Path;
 
+use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
+use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 use crate::sql;
 
 // Re-export key types
@@ -37,6 +39,8 @@ pub enum Error {
     NoConfiguredAddr,
     #[error("Configured address is invalid: {}", _0)]
     InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[error("no data provided")]
+    Empty,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -46,8 +50,9 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// This trait is implemented for rPGP's [SignedPublicKey] and
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
-pub trait DcKey: Serialize + Deserializable {
-    type KeyType: Serialize + Deserializable;
+#[async_trait]
+pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
+    type KeyType: Serialize + Deserializable + KeyTrait + Clone;
 
     /// Create a key from some bytes.
     fn from_slice(bytes: &[u8]) -> Result<Self::KeyType> {
@@ -64,99 +69,189 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
-    /// Load the users' default key from the database.
-    fn load_self(context: &Context) -> Result<Self::KeyType>;
+    /// Create a key from an ASCII-armored string.
+    ///
+    /// Returns the key and a map of any headers which might have been set in
+    /// the ASCII-armored representation.
+    fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
+        let bytes = data.as_bytes();
+        Self::KeyType::from_armor_single(Cursor::new(bytes)).map_err(Error::Pgp)
+    }
 
-    /// Serialise the key to a base64 string.
-    fn to_base64(&self) -> String {
+    /// Load the users' default key from the database.
+    async fn load_self(context: &Context) -> Result<Self::KeyType>;
+
+    /// Serialise the key as bytes.
+    fn to_bytes(&self) -> Vec<u8> {
         // Not using Serialize::to_bytes() to make clear *why* it is
         // safe to ignore this error.
         // Because we write to a Vec<u8> the io::Write impls never
         // fail and we can hide this error.
         let mut buf = Vec::new();
         self.to_writer(&mut buf).unwrap();
-        base64::encode(&buf)
+        buf
+    }
+
+    /// Serialise the key to a base64 string.
+    fn to_base64(&self) -> String {
+        base64::encode(&DcKey::to_bytes(self))
+    }
+
+    /// Serialise the key to ASCII-armored representation.
+    ///
+    /// Each header line must be terminated by `\r\n`.  Only allows setting one
+    /// header as a simplification since that's the only way it's used so far.
+    // Since .to_armored_string() are actual methods on SignedPublicKey and
+    // SignedSecretKey we can not generically implement this.
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String;
+
+    /// The fingerprint for the key.
+    fn fingerprint(&self) -> Fingerprint {
+        Fingerprint::new(KeyTrait::fingerprint(self)).expect("Invalid fingerprint from rpgp")
     }
 }
 
+#[async_trait]
 impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
 
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
+    async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        let res: std::result::Result<Vec<u8>, _> = context
+            .sql
+            .query_value(
+                r#"
             SELECT public_key
               FROM keypairs
              WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
                AND is_default=1;
             "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
+                paramsx![],
+            )
+            .await;
+        match res {
             Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
+            Err(sql::Error::Sqlx(sqlx::Error::RowNotFound)) => {
+                let keypair = generate_keypair(context).await?;
                 Ok(keypair.public)
             }
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to ignore this error.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
 }
 
+#[async_trait]
 impl DcKey for SignedSecretKey {
     type KeyType = SignedSecretKey;
 
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
+    async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        let res: std::result::Result<Vec<u8>, _> = context
+            .sql
+            .query_value(
+                r#"
             SELECT private_key
               FROM keypairs
              WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
                AND is_default=1;
             "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
+                paramsx![],
+            )
+            .await;
+        match res {
             Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
+            Err(sql::Error::Sqlx(sqlx::Error::RowNotFound)) => {
+                let keypair = generate_keypair(context).await?;
                 Ok(keypair.secret)
             }
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to do these unwraps.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.  The string is always ASCII.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
 }
 
-fn generate_keypair(context: &Context) -> Result<KeyPair> {
+/// Deltachat extension trait for secret keys.
+///
+/// Provides some convenience wrappers only applicable to [SignedSecretKey].
+pub trait DcSecretKey {
+    /// Create a public key from a private one.
+    fn split_public_key(&self) -> Result<SignedPublicKey>;
+}
+
+impl DcSecretKey for SignedSecretKey {
+    fn split_public_key(&self) -> Result<SignedPublicKey> {
+        self.verify()?;
+        let unsigned_pubkey = SecretKeyTrait::public_key(self);
+        let signed_pubkey = unsigned_pubkey.sign(self, || "".into())?;
+        Ok(signed_pubkey)
+    }
+}
+
+async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
+        .await
         .ok_or_else(|| Error::NoConfiguredAddr)?;
     let addr = EmailAddress::new(&addr)?;
-    let _guard = context.generating_key_mutex.lock().unwrap();
+    let _guard = context.generating_key_mutex.lock().await;
 
     // Check if the key appeared while we were waiting on the lock.
-    match context.sql.query_row(
-        r#"
+    let res: std::result::Result<(Vec<u8>, Vec<u8>), _> = context
+        .sql
+        .query_row(
+            r#"
         SELECT public_key, private_key
           FROM keypairs
-         WHERE addr=?1
+         WHERE addr=?
            AND is_default=1;
         "#,
-        params![addr],
-        |row| Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?)),
-    ) {
+            paramsx![addr.to_string()],
+        )
+        .await;
+    match res {
         Ok((pub_bytes, sec_bytes)) => Ok(KeyPair {
             addr,
             public: SignedPublicKey::from_slice(&pub_bytes)?,
             secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
-        Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
+        Err(sql::Error::Sqlx(sqlx::Error::RowNotFound)) => {
             let start = std::time::Instant::now();
-            let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType))
+            let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
-            let keypair = crate::pgp::create_keypair(addr, keytype)?;
-            store_self_keypair(context, &keypair, KeyPairUse::Default)?;
+            let keypair =
+                async_std::task::spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
+                    .await?;
+            store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
@@ -168,198 +263,6 @@ fn generate_keypair(context: &Context) -> Result<KeyPair> {
     }
 }
 
-/// Cryptographic key
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum Key {
-    Public(SignedPublicKey),
-    Secret(SignedSecretKey),
-}
-
-impl From<SignedPublicKey> for Key {
-    fn from(key: SignedPublicKey) -> Self {
-        Key::Public(key)
-    }
-}
-
-impl From<SignedSecretKey> for Key {
-    fn from(key: SignedSecretKey) -> Self {
-        Key::Secret(key)
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl Key {
-    pub fn is_public(&self) -> bool {
-        match self {
-            Key::Public(_) => true,
-            Key::Secret(_) => false,
-        }
-    }
-
-    pub fn is_secret(&self) -> bool {
-        !self.is_public()
-    }
-
-    pub fn from_slice(bytes: &[u8], key_type: KeyType) -> Option<Self> {
-        if bytes.is_empty() {
-            return None;
-        }
-        let res: std::result::Result<Key, _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-            KeyType::Private => SignedSecretKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-        };
-
-        match res {
-            Ok(key) => Some(key),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn from_armored_string(
-        data: &str,
-        key_type: KeyType,
-    ) -> Option<(Self, BTreeMap<String, String>)> {
-        let bytes = data.as_bytes();
-        let res: std::result::Result<(Key, _), _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-            KeyType::Private => SignedSecretKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-        };
-
-        match res {
-            Ok(res) => Some(res),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn to_bytes(&self) -> Vec<u8> {
-        match self {
-            Key::Public(k) => k.to_bytes().unwrap_or_default(),
-            Key::Secret(k) => k.to_bytes().unwrap_or_default(),
-        }
-    }
-
-    pub fn verify(&self) -> bool {
-        match self {
-            Key::Public(k) => k.verify().is_ok(),
-            Key::Secret(k) => k.verify().is_ok(),
-        }
-    }
-
-    pub fn to_base64(&self) -> String {
-        let buf = self.to_bytes();
-        base64::encode(&buf)
-    }
-
-    pub fn to_armored_string(
-        &self,
-        headers: Option<&BTreeMap<String, String>>,
-    ) -> pgp::errors::Result<String> {
-        match self {
-            Key::Public(k) => k.to_armored_string(headers),
-            Key::Secret(k) => k.to_armored_string(headers),
-        }
-    }
-
-    /// Each header line must be terminated by `\r\n`
-    pub fn to_asc(&self, header: Option<(&str, &str)>) -> String {
-        let headers = header.map(|(key, value)| {
-            let mut m = BTreeMap::new();
-            m.insert(key.to_string(), value.to_string());
-            m
-        });
-
-        self.to_armored_string(headers.as_ref())
-            .expect("failed to serialize key")
-    }
-
-    pub fn write_asc_to_file(
-        &self,
-        file: impl AsRef<Path>,
-        context: &Context,
-    ) -> std::io::Result<()> {
-        let file_content = self.to_asc(None).into_bytes();
-
-        let res = dc_write_file(context, &file, &file_content);
-        if res.is_err() {
-            error!(context, "Cannot write key to {}", file.as_ref().display());
-        }
-        res
-    }
-
-    pub fn fingerprint(&self) -> String {
-        match self {
-            Key::Public(k) => hex::encode_upper(k.fingerprint()),
-            Key::Secret(k) => hex::encode_upper(k.fingerprint()),
-        }
-    }
-
-    pub fn formatted_fingerprint(&self) -> String {
-        let rawhex = self.fingerprint();
-        dc_format_fingerprint(&rawhex)
-    }
-
-    pub fn split_key(&self) -> Option<Key> {
-        match self {
-            Key::Public(_) => None,
-            Key::Secret(k) => {
-                let pub_key = k.public_key();
-                pub_key.sign(k, || "".into()).map(Key::Public).ok()
-            }
-        }
-    }
-}
-
 /// Use of a [KeyPair] for encryption or decryption.
 ///
 /// This is used by [store_self_keypair] to know what kind of key is
@@ -402,103 +305,130 @@ impl SaveKeyError {
 /// same key again overwrites it.
 ///
 /// [Config::ConfiguredAddr]: crate::config::Config::ConfiguredAddr
-pub fn store_self_keypair(
+pub async fn store_self_keypair(
     context: &Context,
     keypair: &KeyPair,
     default: KeyPairUse,
 ) -> std::result::Result<(), SaveKeyError> {
     // Everything should really be one transaction, more refactoring
     // is needed for that.
-    let public_key = keypair
-        .public
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise public key", err))?;
-    let secret_key = keypair
-        .secret
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise secret key", err))?;
+    let public_key = DcKey::to_bytes(&keypair.public);
+    let secret_key = DcKey::to_bytes(&keypair.secret);
     context
         .sql
         .execute(
             "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
-            params![public_key, secret_key],
+            paramsx![&public_key, &secret_key],
         )
+        .await
         .map_err(|err| SaveKeyError::new("failed to remove old use of key", err))?;
     if default == KeyPairUse::Default {
         context
             .sql
-            .execute("UPDATE keypairs SET is_default=0;", params![])
+            .execute("UPDATE keypairs SET is_default=0;", paramsx![])
+            .await
             .map_err(|err| SaveKeyError::new("failed to clear default", err))?;
     }
     let is_default = match default {
-        KeyPairUse::Default => true,
-        KeyPairUse::ReadOnly => false,
+        KeyPairUse::Default => true as i32,
+        KeyPairUse::ReadOnly => false as i32,
     };
+
+    let addr = keypair.addr.to_string();
+    let t = time();
+
+    let params = paramsx![addr, is_default, public_key, secret_key, t];
     context
         .sql
         .execute(
             "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
                 VALUES (?,?,?,?,?);",
-            params![
-                keypair.addr.to_string(),
-                is_default as i32,
-                public_key,
-                secret_key,
-                time()
-            ],
+            params,
         )
-        .map(|_| ())
-        .map_err(|err| SaveKeyError::new("failed to insert keypair", err))
+        .await
+        .map_err(|err| SaveKeyError::new("failed to insert keypair", err))?;
+
+    Ok(())
 }
 
-/// Make a fingerprint human-readable, in hex format.
-pub fn dc_format_fingerprint(fingerprint: &str) -> String {
-    // split key into chunks of 4 with space, and 20 newline
-    let mut res = String::new();
+/// A key fingerprint
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct Fingerprint(Vec<u8>);
 
-    for (i, c) in fingerprint.chars().enumerate() {
-        if i > 0 && i % 20 == 0 {
-            res += "\n";
-        } else if i > 0 && i % 4 == 0 {
-            res += " ";
+impl Fingerprint {
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+        match v.len() {
+            20 => Ok(Fingerprint(v)),
+            _ => Err(FingerprintError::WrongLength),
         }
-
-        res += &c.to_string();
     }
 
-    res
+    /// Make a hex string from the fingerprint.
+    ///
+    /// Use [std::fmt::Display] or [ToString::to_string] to get a
+    /// human-readable formatted string.
+    pub fn hex(&self) -> String {
+        hex::encode_upper(&self.0)
+    }
 }
 
-/// Bring a human-readable or otherwise formatted fingerprint back to the 40-characters-uppercase-hex format.
-pub fn dc_normalize_fingerprint(fp: &str) -> String {
-    fp.to_uppercase()
-        .chars()
-        .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
-        .collect()
+/// Make a human-readable fingerprint.
+impl fmt::Display for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Split key into chunks of 4 with space and newline at 20 chars
+        for (i, c) in self.hex().chars().enumerate() {
+            if i > 0 && i % 20 == 0 {
+                writeln!(f)?;
+            } else if i > 0 && i % 4 == 0 {
+                write!(f, " ")?;
+            }
+            write!(f, "{}", c)?;
+        }
+        Ok(())
+    }
+}
+
+/// Parse a human-readable or otherwise formatted fingerprint.
+impl std::str::FromStr for Fingerprint {
+    type Err = FingerprintError;
+
+    fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
+        let hex_repr: String = input
+            .to_uppercase()
+            .chars()
+            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .collect();
+        let v: Vec<u8> = hex::decode(hex_repr)?;
+        let fp = Fingerprint::new(v)?;
+        Ok(fp)
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use std::convert::TryFrom;
 
+    use std::error::Error;
+
+    use async_std::sync::Arc;
     use lazy_static::lazy_static;
 
     lazy_static! {
         static ref KEYPAIR: KeyPair = alice_keypair();
     }
 
-    #[test]
-    fn test_normalize_fingerprint() {
-        let fingerprint = dc_normalize_fingerprint(" 1234  567890 \n AbcD abcdef ABCDEF ");
-
-        assert_eq!(fingerprint, "1234567890ABCDABCDEFABCDEF");
-    }
-
     #[test]
     fn test_from_armored_string() {
-        let (private_key, _) = Key::from_armored_string(
+        let (private_key, _) = SignedSecretKey::from_asc(
             "-----BEGIN PGP PRIVATE KEY BLOCK-----
 
 xcLYBF0fgz4BCADnRUV52V4xhSsU56ZaAn3+3oG86MZhXy4X8w14WZZDf0VJGeTh
@@ -556,152 +486,149 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 7yPJeQ==
 =KZk/
 -----END PGP PRIVATE KEY BLOCK-----",
-            KeyType::Private,
         )
-        .expect("failed to decode"); // NOTE: if you take out the ===GU1/ part, everything passes!
-        let binary = private_key.to_bytes();
-        Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        .expect("failed to decode");
+        let binary = DcKey::to_bytes(&private_key);
+        SignedSecretKey::from_slice(&binary).expect("invalid private key");
     }
 
     #[test]
-    fn test_format_fingerprint() {
-        let fingerprint = dc_format_fingerprint("1234567890ABCDABCDEFABCDEF1234567890ABCD");
+    fn test_asc_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedPublicKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
 
-        assert_eq!(
-            fingerprint,
-            "1234 5678 90AB CDAB CDEF\nABCD EF12 3456 7890 ABCD"
-        );
+        let key = KEYPAIR.secret.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedSecretKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
     }
 
     #[test]
     fn test_from_slice_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+        let public_key = KEYPAIR.public.clone();
+        let private_key = KEYPAIR.secret.clone();
 
-        let binary = public_key.to_bytes();
-        let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
+        let binary = DcKey::to_bytes(&public_key);
+        let public_key2 = SignedPublicKey::from_slice(&binary).expect("invalid public key");
         assert_eq!(public_key, public_key2);
 
-        let binary = private_key.to_bytes();
-        let private_key2 = Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        let binary = DcKey::to_bytes(&private_key);
+        let private_key2 = SignedSecretKey::from_slice(&binary).expect("invalid private key");
         assert_eq!(private_key, private_key2);
     }
 
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-
         for i in 0..4096 {
             bad_data[i] = (i & 0xff) as u8;
         }
-
         for j in 0..(4096 / 40) {
-            let bad_key = Key::from_slice(
-                &bad_data[j..j + 4096 / 2 + j],
-                if 0 != j & 1 {
-                    KeyType::Public
-                } else {
-                    KeyType::Private
-                },
-            );
-            assert!(bad_key.is_none());
+            let slice = &bad_data[j..j + 4096 / 2 + j];
+            assert!(SignedPublicKey::from_slice(slice).is_err());
+            assert!(SignedSecretKey::from_slice(slice).is_err());
         }
     }
 
     #[test]
-    fn test_load_self_existing() {
+    fn test_base64_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let base64 = key.to_base64();
+        let key2 = SignedPublicKey::from_base64(&base64).unwrap();
+        assert_eq!(key, key2);
+    }
+
+    #[async_std::test]
+    async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = dummy_context();
-        configure_alice_keypair(&t.ctx);
-        let pubkey = SignedPublicKey::load_self(&t.ctx).unwrap();
+        let t = TestContext::new().await;
+        t.configure_alice().await;
+        let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
         assert_eq!(alice.public, pubkey);
-        let seckey = SignedSecretKey::load_self(&t.ctx).unwrap();
+        let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
         assert_eq!(alice.secret, seckey);
     }
 
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_public() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_load_self_generate_public() {
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
             .unwrap();
-        let key = SignedPublicKey::load_self(&t.ctx);
+        let key = SignedPublicKey::load_self(&t.ctx).await;
         assert!(key.is_ok());
     }
 
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_secret() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_load_self_generate_secret() {
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
             .unwrap();
-        let key = SignedSecretKey::load_self(&t.ctx);
+        let key = SignedSecretKey::load_self(&t.ctx).await;
         assert!(key.is_ok());
     }
 
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_concurrent() {
-        use std::sync::Arc;
+    #[async_std::test]
+    async fn test_load_self_generate_concurrent() {
         use std::thread;
 
-        let t = dummy_context();
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
             .unwrap();
-        let ctx = Arc::new(t.ctx);
-        let ctx0 = Arc::clone(&ctx);
-        let thr0 = thread::spawn(move || SignedPublicKey::load_self(&ctx0));
-        let ctx1 = Arc::clone(&ctx);
-        let thr1 = thread::spawn(move || SignedPublicKey::load_self(&ctx1));
+        let ctx = t.ctx.clone();
+        let ctx0 = ctx.clone();
+        let thr0 =
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
+        let ctx1 = ctx.clone();
+        let thr1 =
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
         let res0 = thr0.join().unwrap();
         let res1 = thr1.join().unwrap();
         assert_eq!(res0.unwrap(), res1.unwrap());
     }
 
-    #[test]
-    fn test_ascii_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
-
-        let s = public_key.to_armored_string(None).unwrap();
-        let (public_key2, _) =
-            Key::from_armored_string(&s, KeyType::Public).expect("invalid public key");
-        assert_eq!(public_key, public_key2);
-
-        let s = private_key.to_armored_string(None).unwrap();
-        println!("{}", &s);
-        let (private_key2, _) =
-            Key::from_armored_string(&s, KeyType::Private).expect("invalid private key");
-        assert_eq!(private_key, private_key2);
-    }
-
     #[test]
     fn test_split_key() {
-        let private_key = Key::from(KEYPAIR.secret.clone());
-        let public_wrapped = private_key.split_key().unwrap();
-        let public = SignedPublicKey::try_from(public_wrapped).unwrap();
-        assert_eq!(public.primary_key, KEYPAIR.public.primary_key);
+        let pubkey = KEYPAIR.secret.split_public_key().unwrap();
+        assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
-    #[test]
-    fn test_save_self_key_twice() {
+    #[async_std::test]
+    async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.
-        let t = dummy_context();
-        let nrows = || {
-            t.ctx
+        let t = TestContext::new().await;
+        let ctx = Arc::new(t.ctx);
+
+        let ctx1 = ctx.clone();
+        let nrows = || async {
+            let val: i32 = ctx1
                 .sql
-                .query_get_value::<_, u32>(&t.ctx, "SELECT COUNT(*) FROM keypairs;", params![])
-                .unwrap()
+                .query_value("SELECT COUNT(*) FROM keypairs;", paramsx![])
+                .await
+                .unwrap();
+            val as usize
         };
-        assert_eq!(nrows(), 0);
-        store_self_keypair(&t.ctx, &KEYPAIR, KeyPairUse::Default).unwrap();
-        assert_eq!(nrows(), 1);
-        store_self_keypair(&t.ctx, &KEYPAIR, KeyPairUse::Default).unwrap();
-        assert_eq!(nrows(), 1);
+        assert_eq!(nrows().await, 0);
+        store_self_keypair(&ctx, &KEYPAIR, KeyPairUse::Default)
+            .await
+            .unwrap();
+        assert_eq!(nrows().await, 1);
+        store_self_keypair(&ctx, &KEYPAIR, KeyPairUse::Default)
+            .await
+            .unwrap();
+        assert_eq!(nrows().await, 1);
     }
 
     // Convenient way to create a new key if you need one, run with
@@ -724,4 +651,49 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     //     )
     //     .unwrap();
     // }
+
+    #[test]
+    fn test_fingerprint_from_str() {
+        let res = Fingerprint::new(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+
+        let fp: Fingerprint = "0102030405060708090A0B0c0d0e0F1011121314".parse().unwrap();
+        assert_eq!(fp, res);
+
+        let fp: Fingerprint = "zzzz 0102 0304 0506\n0708090a0b0c0D0E0F1011121314 yyy"
+            .parse()
+            .unwrap();
+        assert_eq!(fp, res);
+
+        let err = "1".parse::<Fingerprint>().err().unwrap();
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
+    }
+
+    #[test]
+    fn test_fingerprint_hex() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(fp.hex(), "0102040810204080FF0A0B0C0D0E0F1011121314");
+    }
+
+    #[test]
+    fn test_fingerprint_to_string() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(
+            fp.to_string(),
+            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF 1314"
+        );
+    }
 }

src/keyring.rs

@@ -1,45 +1,92 @@
-use std::borrow::Cow;
+//! Keyring to perform rpgp operations with.
+
+use anyhow::Result;
 
-use crate::constants::KeyType;
 use crate::context::Context;
-use crate::key::Key;
-use crate::sql::Sql;
+use crate::key::{self, DcKey};
 
-#[derive(Default, Clone, Debug)]
-pub struct Keyring<'a> {
-    keys: Vec<Cow<'a, Key>>,
+/// An in-memory keyring.
+///
+/// Instances are usually constructed just for the rpgp operation and
+/// short-lived.
+#[derive(Clone, Debug, Default)]
+pub struct Keyring<T>
+where
+    T: DcKey,
+{
+    keys: Vec<T>,
 }
 
-impl<'a> Keyring<'a> {
-    pub fn add_owned(&mut self, key: Key) {
-        self.add(Cow::Owned(key))
+impl<T> Keyring<T>
+where
+    T: DcKey<KeyType = T>,
+{
+    /// New empty keyring.
+    pub fn new() -> Keyring<T> {
+        Keyring { keys: Vec::new() }
     }
 
-    pub fn add_ref(&mut self, key: &'a Key) {
-        self.add(Cow::Borrowed(key))
+    /// Create a new keyring with the the user's secret key loaded.
+    pub async fn new_self(context: &Context) -> Result<Keyring<T>, key::Error> {
+        let mut keyring: Keyring<T> = Keyring::new();
+        keyring.load_self(context).await?;
+        Ok(keyring)
     }
 
-    fn add(&mut self, key: Cow<'a, Key>) {
+    /// Load the user's key into the keyring.
+    pub async fn load_self(&mut self, context: &Context) -> Result<(), key::Error> {
+        self.add(T::load_self(context).await?);
+        Ok(())
+    }
+
+    /// Add a key to the keyring.
+    pub fn add(&mut self, key: T) {
         self.keys.push(key);
     }
 
-    pub fn keys(&self) -> &[Cow<'a, Key>] {
-        &self.keys
+    pub fn len(&self) -> usize {
+        self.keys.len()
     }
 
-    pub fn load_self_private_for_decrypting(
-        &mut self,
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> bool {
-        sql.query_get_value(
-            context,
-            "SELECT private_key FROM keypairs ORDER BY addr=? DESC, is_default DESC;",
-            &[self_addr.as_ref()],
-        )
-        .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Private))
-        .map(|key| self.add_owned(key))
-        .is_some()
+    pub fn is_empty(&self) -> bool {
+        self.keys.is_empty()
+    }
+
+    /// A vector with reference to all the keys in the keyring.
+    pub fn keys(&self) -> &[T] {
+        &self.keys
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::key::{SignedPublicKey, SignedSecretKey};
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_keyring_add_keys() {
+        let alice = alice_keypair();
+        let mut pub_ring: Keyring<SignedPublicKey> = Keyring::new();
+        pub_ring.add(alice.public.clone());
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let mut sec_ring: Keyring<SignedSecretKey> = Keyring::new();
+        sec_ring.add(alice.secret.clone());
+        assert_eq!(sec_ring.keys(), [alice.secret]);
+    }
+
+    #[async_std::test]
+    async fn test_keyring_load_self() {
+        // new_self() implies load_self()
+        let t = TestContext::new().await;
+        t.configure_alice().await;
+        let alice = alice_keypair();
+
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -6,19 +6,22 @@
 extern crate num_derive;
 #[macro_use]
 extern crate smallvec;
-#[macro_use]
-extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 #[macro_use]
 pub mod log;
 #[macro_use]
 pub mod error;
 
+#[cfg(feature = "internals")]
+#[macro_use]
+pub mod sql;
+#[cfg(not(feature = "internals"))]
+#[macro_use]
+mod sql;
+
 pub mod headerdef;
 
 pub(crate) mod events;
@@ -36,9 +39,9 @@ pub mod context;
 mod e2ee;
 mod imap;
 pub mod imex;
+mod scheduler;
 #[macro_use]
 pub mod job;
-mod job_thread;
 pub mod key;
 mod keyring;
 pub mod location;
@@ -56,7 +59,6 @@ pub mod qr;
 pub mod securejoin;
 mod simplify;
 mod smtp;
-pub mod sql;
 pub mod stock;
 mod token;
 #[macro_use]

src/location.rs

@@ -1,5 +1,6 @@
 //! Location handling
 
+use async_std::prelude::*;
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
@@ -10,11 +11,10 @@ use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
 use crate::events::Event;
-use crate::job::{self, job_action_exists, job_add, Job};
+use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
-use crate::sql;
 use crate::stock::StockMessage;
 
 /// Location record
@@ -38,6 +38,34 @@ impl Location {
     }
 }
 
+impl<'a> sqlx::FromRow<'a, sqlx::sqlite::SqliteRow> for Location {
+    fn from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Self, sqlx::Error> {
+        use sqlx::Row;
+
+        let msg_id = row.try_get::<i32, _>("msg_id")? as u32;
+        let txt: String = row.try_get("txt")?;
+        let marker = if msg_id != 0 && is_marker(&txt) {
+            Some(txt)
+        } else {
+            None
+        };
+        let loc = Location {
+            location_id: row.try_get::<i32, _>("id")? as u32,
+            latitude: row.try_get("latitude")?,
+            longitude: row.try_get("longitude")?,
+            accuracy: row.try_get("accuracy")?,
+            timestamp: row.try_get("timestamp")?,
+            independent: row.try_get::<i32, _>("independent")? as u32,
+            msg_id,
+            contact_id: row.try_get::<i32, _>("from_id")? as u32,
+            chat_id: row.try_get("chat_id")?,
+            marker,
+        };
+
+        Ok(loc)
+    }
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct Kml {
     pub addr: Option<String>,
@@ -191,114 +219,135 @@ impl Kml {
 }
 
 // location streaming
-pub fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
+pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
     if !(seconds < 0 || chat_id.is_special()) {
-        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
-        if sql::execute(
-            context,
-            &context.sql,
-            "UPDATE chats    \
-             SET locations_send_begin=?,        \
-             locations_send_until=?  \
-             WHERE id=?",
-            params![
-                if 0 != seconds { now } else { 0 },
-                if 0 != seconds { now + seconds } else { 0 },
-                chat_id,
-            ],
-        )
-        .is_ok()
+        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id).await;
+        if context
+            .sql
+            .execute(
+                r#"
+UPDATE chats
+  SET locations_send_begin=?,
+      locations_send_until=?
+  WHERE id=?
+"#,
+                paramsx![
+                    if 0 != seconds { now } else { 0 },
+                    if 0 != seconds { now + seconds } else { 0 },
+                    chat_id
+                ],
+            )
+            .await
+            .is_ok()
         {
             if 0 != seconds && !is_sending_locations_before {
                 let mut msg = Message::new(Viewtype::Text);
-                msg.text =
-                    Some(context.stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0));
-                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
-                chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
-            } else if 0 == seconds && is_sending_locations_before {
-                let stock_str =
-                    context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                chat::add_info_msg(context, chat_id, stock_str);
-            }
-            context.call_cb(Event::ChatModified(chat_id));
-            if 0 != seconds {
-                schedule_MAYBE_SEND_LOCATIONS(context, false);
-                job_add(
-                    context,
-                    job::Action::MaybeSendLocationsEnded,
-                    chat_id.to_u32() as i32,
-                    Params::new(),
-                    seconds + 1,
+                msg.text = Some(
+                    context
+                        .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
+                        .await,
                 );
+                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
+                chat::send_msg(context, chat_id, &mut msg)
+                    .await
+                    .unwrap_or_default();
+            } else if 0 == seconds && is_sending_locations_before {
+                let stock_str = context
+                    .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
+                    .await;
+                chat::add_info_msg(context, chat_id, stock_str).await;
+            }
+            context.emit_event(Event::ChatModified(chat_id));
+            if 0 != seconds {
+                schedule_maybe_send_locations(context, false).await;
+                job::add(
+                    context,
+                    job::Job::new(
+                        job::Action::MaybeSendLocationsEnded,
+                        chat_id.to_u32(),
+                        Params::new(),
+                        seconds + 1,
+                    ),
+                )
+                .await;
             }
         }
     }
 }
 
-#[allow(non_snake_case)]
-fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, force_schedule: bool) {
-    if force_schedule || !job_action_exists(context, job::Action::MaybeSendLocations) {
-        job_add(
+async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) {
+    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await {
+        job::add(
             context,
-            job::Action::MaybeSendLocations,
-            0,
-            Params::new(),
-            60,
-        );
+            job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
+        )
+        .await;
     };
 }
 
-pub fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
+pub async fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
     context
         .sql
         .exists(
             "SELECT id  FROM chats  WHERE (? OR id=?)   AND locations_send_until>?;",
-            params![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
+            paramsx![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
         )
+        .await
         .unwrap_or_default()
 }
 
-pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
+pub async fn set(
+    context: &Context,
+    latitude: f64,
+    longitude: f64,
+    accuracy: f64,
+) -> Result<bool, Error> {
     if latitude == 0.0 && longitude == 0.0 {
-        return true;
+        return Ok(true);
     }
+
     let mut continue_streaming = false;
+    let pool = context.sql.get_pool().await?;
+    let mut rows = sqlx::query_as("SELECT id FROM chats WHERE locations_send_until>?;")
+        .bind(time())
+        .fetch(&pool);
 
-    if let Ok(chats) = context.sql.query_map(
-        "SELECT id FROM chats WHERE locations_send_until>?;",
-        params![time()],
-        |row| row.get::<_, i32>(0),
-        |chats| chats.collect::<Result<Vec<_>, _>>().map_err(Into::into),
-    ) {
-        for chat_id in chats {
-            if let Err(err) = context.sql.execute(
-                    "INSERT INTO locations  \
-                     (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);",
-                    params![
-                        latitude,
-                        longitude,
-                        accuracy,
-                        time(),
-                        chat_id,
-                        DC_CONTACT_ID_SELF,
-                    ]
-            ) {
-                warn!(context, "failed to store location {:?}", err);
-            } else {
-                continue_streaming = true;
-            }
+    while let Some(row) = rows.next().await {
+        let (chat_id,): (i64,) = row?;
+
+        if let Err(err) = context
+            .sql
+            .execute(
+                "INSERT INTO locations  \
+             (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);",
+                paramsx![
+                    latitude,
+                    longitude,
+                    accuracy,
+                    time(),
+                    chat_id,
+                    DC_CONTACT_ID_SELF as i32
+                ],
+            )
+            .await
+        {
+            warn!(context, "failed to store location {:?}", err);
+        } else {
+            continue_streaming = true;
         }
-        if continue_streaming {
-            context.call_cb(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
-        };
-        schedule_MAYBE_SEND_LOCATIONS(context, false);
     }
 
-    continue_streaming
+    if continue_streaming {
+        context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+    };
+
+    schedule_maybe_send_locations(context, false).await;
+
+    Ok(continue_streaming)
 }
 
-pub fn get_range(
+pub async fn get_range(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
@@ -308,16 +357,21 @@ pub fn get_range(
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
+
     context
         .sql
-        .query_map(
-            "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
-             COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt \
-             FROM locations l  LEFT JOIN msgs m ON l.id=m.location_id  WHERE (? OR l.chat_id=?) \
-             AND (? OR l.from_id=?) \
-             AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
-             ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
-            params![
+        .query_rows(
+            r#"
+SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent,
+    COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt
+  FROM locations l
+  LEFT JOIN msgs m ON l.id=m.location_id
+    WHERE (? OR l.chat_id=?)
+      AND (? OR l.from_id=?)
+      AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?))
+  ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;
+"#,
+            paramsx![
                 if chat_id.is_unset() { 1 } else { 0 },
                 chat_id,
                 if contact_id == 0 { 1 } else { 0 },
@@ -325,37 +379,8 @@ pub fn get_range(
                 timestamp_from,
                 timestamp_to,
             ],
-            |row| {
-                let msg_id = row.get(6)?;
-                let txt: String = row.get(9)?;
-                let marker = if msg_id != 0 && is_marker(&txt) {
-                    Some(txt)
-                } else {
-                    None
-                };
-                let loc = Location {
-                    location_id: row.get(0)?,
-                    latitude: row.get(1)?,
-                    longitude: row.get(2)?,
-                    accuracy: row.get(3)?,
-                    timestamp: row.get(4)?,
-                    independent: row.get(5)?,
-                    msg_id,
-                    contact_id: row.get(7)?,
-                    chat_id: row.get(8)?,
-                    marker,
-                };
-                Ok(loc)
-            },
-            |locations| {
-                let mut ret = Vec::new();
-
-                for location in locations {
-                    ret.push(location?);
-                }
-                Ok(ret)
-            },
         )
+        .await
         .unwrap_or_default()
 }
 
@@ -364,28 +389,27 @@ fn is_marker(txt: &str) -> bool {
 }
 
 /// Deletes all locations from the database.
-pub fn delete_all(context: &Context) -> Result<(), Error> {
-    sql::execute(context, &context.sql, "DELETE FROM locations;", params![])?;
-    context.call_cb(Event::LocationChanged(None));
+pub async fn delete_all(context: &Context) -> Result<(), Error> {
+    context
+        .sql
+        .execute("DELETE FROM locations;", paramsx![])
+        .await?;
+    context.emit_event(Event::LocationChanged(None));
     Ok(())
 }
 
-pub fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
+pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
     let mut last_added_location_id = 0;
 
     let self_addr = context
         .get_config(Config::ConfiguredAddr)
+        .await
         .unwrap_or_default();
 
-    let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
+    let (locations_send_begin, locations_send_until, locations_last_sent): (i64, i64, i64) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
-        params![chat_id], |row| {
-            let send_begin: i64 = row.get(0)?;
-            let send_until: i64 = row.get(1)?;
-            let last_sent: i64 = row.get(2)?;
-
-            Ok((send_begin, send_until, last_sent))
-        })?;
+        paramsx![chat_id]
+    ).await?;
 
     let now = time();
     let mut location_count = 0;
@@ -396,40 +420,41 @@ pub fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Erro
             self_addr,
         );
 
-        context.sql.query_map(
-            "SELECT id, latitude, longitude, accuracy, timestamp \
-             FROM locations  WHERE from_id=? \
-             AND timestamp>=? \
-             AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
-             AND independent=0 \
-             GROUP BY timestamp \
-             ORDER BY timestamp;",
-            params![DC_CONTACT_ID_SELF, locations_send_begin, locations_last_sent, DC_CONTACT_ID_SELF],
-            |row| {
-                let location_id: i32 = row.get(0)?;
-                let latitude: f64 = row.get(1)?;
-                let longitude: f64 = row.get(2)?;
-                let accuracy: f64 = row.get(3)?;
-                let timestamp = get_kml_timestamp(row.get(4)?);
+        let pool = context.sql.get_pool().await?;
+        let mut rows = sqlx::query_as(
+            r#"
+SELECT id, latitude, longitude, accuracy, timestamp
+  FROM locations
+  WHERE from_id=?
+    AND timestamp>=?
+    AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?))
+    AND independent=0
+  GROUP BY timestamp
+  ORDER BY timestamp;
+"#,
+        )
+        .bind(DC_CONTACT_ID_SELF as i32)
+        .bind(locations_send_begin)
+        .bind(locations_last_sent)
+        .bind(DC_CONTACT_ID_SELF as i32)
+        .fetch(&pool);
+
+        while let Some(row) = rows.next().await {
+            let (location_id, latitude, longitude, accuracy, timestamp): (i32, f64, f64, f64, i64) =
+                row?;
+            let timestamp = get_kml_timestamp(timestamp);
+
+            ret += &format!(
+                "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
+                timestamp,
+                accuracy,
+                longitude,
+                latitude
+            );
+            location_count += 1;
+            last_added_location_id = location_id as u32;
+        }
 
-                Ok((location_id, latitude, longitude, accuracy, timestamp))
-            },
-            |rows| {
-                for row in rows {
-                    let (location_id, latitude, longitude, accuracy, timestamp) = row?;
-                    ret += &format!(
-                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
-                        timestamp,
-                        accuracy,
-                        longitude,
-                        latitude
-                    );
-                    location_count += 1;
-                    last_added_location_id = location_id as u32;
-                }
-                Ok(())
-            }
-        )?;
         ret += "</Document>\n</kml>";
     }
 
@@ -462,37 +487,38 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
     )
 }
 
-pub fn set_kml_sent_timestamp(
+pub async fn set_kml_sent_timestamp(
     context: &Context,
     chat_id: ChatId,
     timestamp: i64,
 ) -> Result<(), Error> {
-    sql::execute(
-        context,
-        &context.sql,
-        "UPDATE chats SET locations_last_sent=? WHERE id=?;",
-        params![timestamp, chat_id],
-    )?;
-
+    context
+        .sql
+        .execute(
+            "UPDATE chats SET locations_last_sent=? WHERE id=?;",
+            paramsx![timestamp, chat_id],
+        )
+        .await?;
     Ok(())
 }
 
-pub fn set_msg_location_id(
+pub async fn set_msg_location_id(
     context: &Context,
     msg_id: MsgId,
     location_id: u32,
 ) -> Result<(), Error> {
-    sql::execute(
-        context,
-        &context.sql,
-        "UPDATE msgs SET location_id=? WHERE id=?;",
-        params![location_id, msg_id],
-    )?;
+    context
+        .sql
+        .execute(
+            "UPDATE msgs SET location_id=? WHERE id=?;",
+            paramsx![location_id as i32, msg_id],
+        )
+        .await?;
 
     Ok(())
 }
 
-pub fn save(
+pub async fn save(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
@@ -500,54 +526,69 @@ pub fn save(
     independent: bool,
 ) -> Result<u32, Error> {
     ensure!(!chat_id.is_special(), "Invalid chat id");
-    context
-        .sql
-        .prepare2(
-            "SELECT id FROM locations WHERE timestamp=? AND from_id=?",
-            "INSERT INTO locations\
-             (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent) \
-             VALUES (?,?,?,?,?,?,?);",
-            |mut stmt_test, mut stmt_insert, conn| {
-                let mut newest_timestamp = 0;
-                let mut newest_location_id = 0;
 
-                for location in locations {
-                    let exists =
-                        stmt_test.exists(params![location.timestamp, contact_id as i32])?;
+    let mut newest_timestamp = 0;
+    let mut newest_location_id = 0;
 
-                    if independent || !exists {
-                        stmt_insert.execute(params![
-                            location.timestamp,
-                            contact_id as i32,
-                            chat_id,
-                            location.latitude,
-                            location.longitude,
-                            location.accuracy,
-                            independent,
-                        ])?;
+    for location in locations {
+        let &Location {
+            timestamp,
+            latitude,
+            longitude,
+            accuracy,
+            ..
+        } = location;
 
-                        if location.timestamp > newest_timestamp {
-                            newest_timestamp = location.timestamp;
-                            newest_location_id = sql::get_rowid2_with_conn(
-                                context,
-                                conn,
-                                "locations",
-                                "timestamp",
-                                location.timestamp,
-                                "from_id",
-                                contact_id as i32,
-                            );
-                        }
-                    }
-                }
-                Ok(newest_location_id)
-            },
-        )
-        .map_err(Into::into)
+        let exists: Option<i32> = context
+            .sql
+            .query_value_optional(
+                "SELECT id FROM locations WHERE timestamp=? AND from_id=?",
+                paramsx![timestamp, contact_id as i32],
+            )
+            .await?;
+
+        if independent || exists.is_none() {
+            context
+                .sql
+                .execute(
+                    r#"
+INSERT INTO locations
+    (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent)
+  VALUES (?,?,?,?,?,?,?);
+"#,
+                    paramsx![
+                        timestamp,
+                        contact_id as i32,
+                        chat_id,
+                        latitude,
+                        longitude,
+                        accuracy,
+                        independent
+                    ],
+                )
+                .await?;
+
+            if timestamp > newest_timestamp {
+                // okay to drop, as we use cached prepared statements
+                newest_timestamp = timestamp;
+                newest_location_id = context
+                    .sql
+                    .get_rowid2(
+                        "locations",
+                        "timestamp",
+                        timestamp,
+                        "from_id",
+                        contact_id as i32,
+                    )
+                    .await?;
+            }
+        }
+    }
+
+    Ok(newest_location_id)
 }
 
-#[allow(non_snake_case)]
-pub(crate) fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
+pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
     let now = time();
     let mut continue_streaming = false;
     info!(
@@ -555,101 +596,108 @@ pub(crate) fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Statu
         " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
     );
 
-    if let Ok(rows) = context.sql.query_map(
-        "SELECT id, locations_send_begin, locations_last_sent \
-         FROM chats \
-         WHERE locations_send_until>?;",
-        params![now],
-        |row| {
-            let chat_id: ChatId = row.get(0)?;
-            let locations_send_begin: i64 = row.get(1)?;
-            let locations_last_sent: i64 = row.get(2)?;
-            continue_streaming = true;
+    let pool = match context.sql.get_pool().await {
+        Ok(pool) => pool,
+        Err(err) => {
+            return job::Status::Finished(Err(err.into()));
+        }
+    };
 
-            // be a bit tolerant as the timer may not align exactly with time(NULL)
-            if now - locations_last_sent < (60 - 3) {
-                Ok(None)
-            } else {
-                Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
+    let mut rows = sqlx::query_as(
+        r#"
+SELECT id, locations_send_begin, locations_last_sent
+  FROM chats
+  WHERE locations_send_until>?;
+"#,
+    )
+    .bind(now)
+    .fetch(&pool);
+
+    while let Some(row) = rows.next().await {
+        let (chat_id, locations_send_begin, locations_last_sent): (ChatId, i64, i64) = match row {
+            Ok(res) => res,
+            Err(err) => {
+                warn!(context, "invalid row: {}", err);
+                continue;
             }
-        },
-        |rows| {
-            rows.filter_map(|v| v.transpose())
-                .collect::<Result<Vec<_>, _>>()
-                .map_err(Into::into)
-        },
-    ) {
-        let msgs = context
-            .sql
-            .prepare(
-                "SELECT id \
-                 FROM locations \
-                 WHERE from_id=? \
-                 AND timestamp>=? \
-                 AND timestamp>? \
-                 AND independent=0 \
-                 ORDER BY timestamp;",
-                |mut stmt_locations, _| {
-                    let msgs = rows
-                        .into_iter()
-                        .filter_map(|(chat_id, locations_send_begin, locations_last_sent)| {
-                            if !stmt_locations
-                                .exists(params![
-                                    DC_CONTACT_ID_SELF,
-                                    locations_send_begin,
-                                    locations_last_sent,
-                                ])
-                                .unwrap_or_default()
-                            {
-                                // if there is no new location, there's nothing to send.
-                                // however, maybe we want to bypass this test eg. 15 minutes
-                                None
-                            } else {
-                                // pending locations are attached automatically to every message,
-                                // so also to this empty text message.
-                                // DC_CMD_LOCATION is only needed to create a nicer subject.
-                                //
-                                // for optimisation and to avoid flooding the sending queue,
-                                // we could sending these messages only if we're really online.
-                                // the easiest way to determine this, is to check for an empty message queue.
-                                // (might not be 100%, however, as positions are sent combined later
-                                // and dc_set_location() is typically called periodically, this is ok)
-                                let mut msg = Message::new(Viewtype::Text);
-                                msg.hidden = true;
-                                msg.param.set_cmd(SystemMessage::LocationOnly);
-                                Some((chat_id, msg))
-                            }
-                        })
-                        .collect::<Vec<_>>();
-                    Ok(msgs)
-                },
-            )
-            .unwrap_or_default(); // TODO: Better error handling
+        };
+        continue_streaming = true;
+
+        // be a bit tolerant as the timer may not align exactly with time(NULL)
+        if now - locations_last_sent < (60 - 3) {
+            continue;
+        }
+        let exists = context
+            .sql
+            .exists(
+                r#"
+SELECT id
+  FROM locations
+  WHERE from_id=?
+    AND timestamp>=?
+    AND timestamp>?
+    AND independent=0
+  ORDER BY timestamp;",
+"#,
+                paramsx![
+                    DC_CONTACT_ID_SELF as i32,
+                    locations_send_begin,
+                    locations_last_sent,
+                ],
+            )
+            .await
+            .unwrap_or_default();
+
+        if !exists {
+            // if there is no new location, there's nothing to send.
+            // however, maybe we want to bypass this test eg. 15 minutes
+        } else {
+            // pending locations are attached automatically to every message,
+            // so also to this empty text message.
+            // DC_CMD_LOCATION is only needed to create a nicer subject.
+            //
+            // for optimisation and to avoid flooding the sending queue,
+            // we could sending these messages only if we're really online.
+            // the easiest way to determine this, is to check for an empty message queue.
+            // (might not be 100%, however, as positions are sent combined later
+            // and dc_set_location() is typically called periodically, this is ok)
+            let mut msg = Message::new(Viewtype::Text);
+            msg.hidden = true;
+            msg.param.set_cmd(SystemMessage::LocationOnly);
 
-        for (chat_id, mut msg) in msgs.into_iter() {
             // TODO: better error handling
-            chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
+            chat::send_msg(context, chat_id, &mut msg)
+                .await
+                .unwrap_or_default();
         }
     }
+
     if continue_streaming {
-        schedule_MAYBE_SEND_LOCATIONS(context, true);
+        schedule_maybe_send_locations(context, true).await;
     }
+
     job::Status::Finished(Ok(()))
 }
 
-#[allow(non_snake_case)]
-pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> job::Status {
+pub(crate) async fn job_maybe_send_locations_ended(
+    context: &Context,
+    job: &mut Job,
+) -> job::Status {
     // this function is called when location-streaming _might_ have ended for a chat.
     // the function checks, if location-streaming is really ended;
     // if so, a device-message is added if not yet done.
 
     let chat_id = ChatId::new(job.foreign_id);
 
-    let (send_begin, send_until) = job_try!(context.sql.query_row(
-        "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
-        params![chat_id],
-        |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
-    ));
+    let (send_begin, send_until): (i64, i64) = job_try!(
+        context
+            .sql
+            .query_row(
+                "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
+                paramsx![chat_id],
+            )
+            .await
+    );
 
     if !(send_begin != 0 && time() <= send_until) {
         // still streaming -
@@ -658,13 +706,15 @@ pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> jo
         if !(send_begin == 0 && send_until == 0) {
             // not streaming, device-message already sent
             job_try!(context.sql.execute(
-                "UPDATE chats    SET locations_send_begin=0, locations_send_until=0  WHERE id=?",
-                params![chat_id],
-            ));
+                "UPDATE chats SET locations_send_begin=0, locations_send_until=0  WHERE id=?",
+                paramsx![chat_id],
+            ).await);
 
-            let stock_str = context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-            chat::add_info_msg(context, chat_id, stock_str);
-            context.call_cb(Event::ChatModified(chat_id));
+            let stock_str = context
+                .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
+                .await;
+            chat::add_info_msg(context, chat_id, stock_str).await;
+            context.emit_event(Event::ChatModified(chat_id));
         }
     }
     job::Status::Finished(Ok(()))
@@ -673,11 +723,11 @@ pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> jo
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
-    #[test]
-    fn test_kml_parse() {
-        let context = dummy_context();
+    #[async_std::test]
+    async fn test_kml_parse() {
+        let context = TestContext::new().await;
 
         let xml =
             b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";

src/log.rs

@@ -7,9 +7,7 @@ macro_rules! info {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        let thread = ::std::thread::current();
-        let full = format!("{thid:?} {file}:{line}: {msg}",
-                           thid = thread.id(),
+        let full = format!("{file}:{line}: {msg}",
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
@@ -24,9 +22,7 @@ macro_rules! warn {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        let thread = ::std::thread::current();
-        let full = format!("{thid:?} {file}:{line}: {msg}",
-                           thid = thread.id(),
+        let full = format!("{file}:{line}: {msg}",
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
@@ -48,6 +44,6 @@ macro_rules! error {
 #[macro_export]
 macro_rules! emit_event {
     ($ctx:expr, $event:expr) => {
-        $ctx.call_cb($event);
+        $ctx.emit_event($event);
     };
 }

src/login_param.rs

@@ -50,59 +50,60 @@ impl LoginParam {
     }
 
     /// Read the login parameters from the database.
-    pub fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
+    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
         let key = format!("{}addr", prefix);
         let addr = sql
-            .get_raw_config(context, key)
+            .get_raw_config(key)
+            .await
             .unwrap_or_default()
             .trim()
             .to_string();
 
         let key = format!("{}mail_server", prefix);
-        let mail_server = sql.get_raw_config(context, key).unwrap_or_default();
+        let mail_server = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}mail_port", prefix);
-        let mail_port = sql.get_raw_config_int(context, key).unwrap_or_default();
+        let mail_port = sql.get_raw_config_int(key).await.unwrap_or_default();
 
         let key = format!("{}mail_user", prefix);
-        let mail_user = sql.get_raw_config(context, key).unwrap_or_default();
+        let mail_user = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}mail_pw", prefix);
-        let mail_pw = sql.get_raw_config(context, key).unwrap_or_default();
+        let mail_pw = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}imap_certificate_checks", prefix);
         let imap_certificate_checks =
-            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+            if let Some(certificate_checks) = sql.get_raw_config_int(key).await {
                 num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
             } else {
                 Default::default()
             };
 
         let key = format!("{}send_server", prefix);
-        let send_server = sql.get_raw_config(context, key).unwrap_or_default();
+        let send_server = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}send_port", prefix);
-        let send_port = sql.get_raw_config_int(context, key).unwrap_or_default();
+        let send_port = sql.get_raw_config_int(key).await.unwrap_or_default();
 
         let key = format!("{}send_user", prefix);
-        let send_user = sql.get_raw_config(context, key).unwrap_or_default();
+        let send_user = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}send_pw", prefix);
-        let send_pw = sql.get_raw_config(context, key).unwrap_or_default();
+        let send_pw = sql.get_raw_config(key).await.unwrap_or_default();
 
         let key = format!("{}smtp_certificate_checks", prefix);
         let smtp_certificate_checks =
-            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+            if let Some(certificate_checks) = sql.get_raw_config_int(key).await {
                 num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
             } else {
                 Default::default()
             };
 
         let key = format!("{}server_flags", prefix);
-        let server_flags = sql.get_raw_config_int(context, key).unwrap_or_default();
+        let server_flags = sql.get_raw_config_int(key).await.unwrap_or_default();
 
         LoginParam {
             addr,
@@ -120,12 +121,8 @@ impl LoginParam {
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
-    pub fn save_to_database(
+    pub async fn save_to_database(
         &self,
         context: &Context,
         prefix: impl AsRef<str>,
@@ -134,40 +131,49 @@ impl LoginParam {
         let sql = &context.sql;
 
         let key = format!("{}addr", prefix);
-        sql.set_raw_config(context, key, Some(&self.addr))?;
+        sql.set_raw_config(context, key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_server))?;
+        sql.set_raw_config(context, key, Some(&self.mail_server))
+            .await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(context, key, self.mail_port)?;
+        sql.set_raw_config_int(context, key, self.mail_port).await?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_user))?;
+        sql.set_raw_config(context, key, Some(&self.mail_user))
+            .await?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_pw))?;
+        sql.set_raw_config(context, key, Some(&self.mail_pw))
+            .await?;
 
         let key = format!("{}imap_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)?;
+        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)
+            .await?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_server))?;
+        sql.set_raw_config(context, key, Some(&self.send_server))
+            .await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(context, key, self.send_port)?;
+        sql.set_raw_config_int(context, key, self.send_port).await?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_user))?;
+        sql.set_raw_config(context, key, Some(&self.send_user))
+            .await?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_pw))?;
+        sql.set_raw_config(context, key, Some(&self.send_pw))
+            .await?;
 
         let key = format!("{}smtp_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)?;
+        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)
+            .await?;
 
         let key = format!("{}server_flags", prefix);
-        sql.set_raw_config_int(context, key, self.server_flags)?;
+        sql.set_raw_config_int(context, key, self.server_flags)
+            .await?;
 
         Ok(())
     }
@@ -258,21 +264,15 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-    match certificate_checks {
-        CertificateChecks::Automatic => {
-            // Same as AcceptInvalidCertificates for now.
-            // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
-        }
-        CertificateChecks::Strict => tls_builder,
-        CertificateChecks::AcceptInvalidCertificates
-        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
+
+    if strict_tls {
+        tls_builder
+    } else {
+        tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
+            .danger_accept_invalid_certs(true)
     }
 }
 

src/lot.rs

@@ -1,4 +1,6 @@
-use deltachat_derive::{FromSql, ToSql};
+use deltachat_derive::*;
+
+use crate::key::Fingerprint;
 
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
@@ -14,13 +16,13 @@ pub struct Lot {
     pub(crate) timestamp: i64,
     pub(crate) state: LotState,
     pub(crate) id: u32,
-    pub(crate) fingerprint: Option<String>,
+    pub(crate) fingerprint: Option<Fingerprint>,
     pub(crate) invitenumber: Option<String>,
     pub(crate) auth: Option<String>,
 }
 
 #[repr(u8)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, Sqlx)]
 pub enum Meaning {
     None = 0,
     Text1Draft = 1,
@@ -65,7 +67,7 @@ impl Lot {
 }
 
 #[repr(i32)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, Sqlx)]
 pub enum LotState {
     // Default
     Undefined = 0,

src/mimefactory.rs

@@ -1,3 +1,4 @@
+use async_std::prelude::*;
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
@@ -66,74 +67,82 @@ pub struct RenderedEmail {
 }
 
 impl<'a, 'b> MimeFactory<'a, 'b> {
-    pub fn from_msg(
+    pub async fn from_msg(
         context: &'a Context,
         msg: &'b Message,
         attach_selfavatar: bool,
     ) -> Result<MimeFactory<'a, 'b>, Error> {
-        let chat = Chat::load_from_db(context, msg.chat_id)?;
+        let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
         let from_addr = context
             .get_config(Config::ConfiguredAddr)
+            .await
+            .unwrap_or_default();
+        let from_displayname = context
+            .get_config(Config::Displayname)
+            .await
             .unwrap_or_default();
-        let from_displayname = context.get_config(Config::Displayname).unwrap_or_default();
         let mut recipients = Vec::with_capacity(5);
         let mut req_mdn = false;
 
         if chat.is_self_talk() {
             recipients.push((from_displayname.to_string(), from_addr.to_string()));
         } else {
-            context.sql.query_map(
-                "SELECT c.authname, c.addr  \
-                 FROM chats_contacts cc  \
-                 LEFT JOIN contacts c ON cc.contact_id=c.id  \
-                 WHERE cc.chat_id=? AND cc.contact_id>9;",
-                params![msg.chat_id],
-                |row| {
-                    let authname: String = row.get(0)?;
-                    let addr: String = row.get(1)?;
-                    Ok((authname, addr))
-                },
-                |rows| {
-                    for row in rows {
-                        let (authname, addr) = row?;
-                        if !recipients_contain_addr(&recipients, &addr) {
-                            recipients.push((authname, addr));
-                        }
-                    }
-                    Ok(())
-                },
-            )?;
+            let pool = context.sql.get_pool().await?;
+
+            let mut rows = sqlx::query_as(
+                r#"
+SELECT c.authname, c.addr
+  FROM chats_contacts cc
+  LEFT JOIN contacts c ON cc.contact_id=c.id
+    WHERE cc.chat_id=? AND cc.contact_id>9;
+"#,
+            )
+            .bind(msg.chat_id)
+            .fetch(&pool);
+
+            while let Some(row) = rows.next().await {
+                let (authname, addr): (String, String) = row?;
+
+                if !recipients_contain_addr(&recipients, &addr) {
+                    recipients.push((authname, addr));
+                }
+            }
 
             let command = msg.param.get_cmd();
 
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
-                && context.get_config_bool(Config::MdnsEnabled)
+                && context.get_config_bool(Config::MdnsEnabled).await
             {
                 req_mdn = true;
             }
         }
-        let (in_reply_to, references) = context.sql.query_row(
-            "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
-            params![msg.id],
-            |row| {
-                let in_reply_to: String = row.get(0)?;
-                let references: String = row.get(1)?;
-
-                Ok((
+        let (in_reply_to, references) = context
+            .sql
+            .query_row(
+                "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
+                paramsx![msg.id],
+            )
+            .await
+            .map(|(in_reply_to, references): (String, String)| {
+                (
                     render_rfc724_mid_list(&in_reply_to),
                     render_rfc724_mid_list(&references),
-                ))
-            },
-        )?;
+                )
+            })?;
 
+        let default_str = context
+            .stock_str(StockMessage::StatusLine)
+            .await
+            .to_string();
         let factory = MimeFactory {
             from_addr,
             from_displayname,
             selfstatus: context
                 .get_config(Config::Selfstatus)
-                .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
+                .await
+                .unwrap_or_else(|| default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -148,29 +157,42 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         Ok(factory)
     }
 
-    pub fn from_mdn(
+    pub async fn from_mdn(
         context: &'a Context,
         msg: &'b Message,
         additional_msg_ids: Vec<String>,
-    ) -> Result<Self, Error> {
+    ) -> Result<MimeFactory<'a, 'b>, Error> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
-        let contact = Contact::load_from_db(context, msg.from_id)?;
+        let contact = Contact::load_from_db(context, msg.from_id).await?;
+        let from_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await
+            .unwrap_or_default();
+        let from_displayname = context
+            .get_config(Config::Displayname)
+            .await
+            .unwrap_or_default();
+        let default_str = context
+            .stock_str(StockMessage::StatusLine)
+            .await
+            .to_string();
+        let selfstatus = context
+            .get_config(Config::Selfstatus)
+            .await
+            .unwrap_or_else(|| default_str);
+        let timestamp = dc_create_smeared_timestamp(context).await;
 
-        Ok(MimeFactory {
+        let res = MimeFactory::<'a, 'b> {
             context,
-            from_addr: context
-                .get_config(Config::ConfiguredAddr)
-                .unwrap_or_default(),
-            from_displayname: context.get_config(Config::Displayname).unwrap_or_default(),
-            selfstatus: context
-                .get_config(Config::Selfstatus)
-                .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
+            from_addr,
+            from_displayname,
+            selfstatus,
             recipients: vec![(
                 contact.get_authname().to_string(),
                 contact.get_addr().to_string(),
             )],
-            timestamp: dc_create_smeared_timestamp(context),
+            timestamp,
             loaded: Loaded::MDN { additional_msg_ids },
             msg,
             in_reply_to: String::default(),
@@ -178,26 +200,31 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             req_mdn: false,
             last_added_location_id: 0,
             attach_selfavatar: false,
-        })
+        };
+
+        Ok(res)
     }
 
-    fn peerstates_for_recipients(&self) -> Result<Vec<(Option<Peerstate>, &str)>, Error> {
+    async fn peerstates_for_recipients(&self) -> Result<Vec<(Option<Peerstate>, &str)>, Error> {
         let self_addr = self
             .context
             .get_config(Config::ConfiguredAddr)
+            .await
             .ok_or_else(|| format_err!("Not configured"))?;
 
-        Ok(self
+        let mut res = Vec::new();
+        for (_, addr) in self
             .recipients
             .iter()
             .filter(|(_, addr)| addr != &self_addr)
-            .map(|(_, addr)| {
-                (
-                    Peerstate::from_addr(self.context, &self.context.sql, addr),
-                    addr.as_str(),
-                )
-            })
-            .collect())
+        {
+            res.push((
+                Peerstate::from_addr(self.context, addr).await.ok(),
+                addr.as_str(),
+            ));
+        }
+
+        Ok(res)
     }
 
     fn is_e2ee_guaranteed(&self) -> bool {
@@ -257,11 +284,11 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn should_do_gossip(&self) -> bool {
+    async fn should_do_gossip(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
-                let gossiped_timestamp = chat.get_gossiped_timestamp(self.context);
+                let gossiped_timestamp = chat.get_gossiped_timestamp(self.context).await;
                 if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
                     return true;
                 }
@@ -302,12 +329,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn subject_str(&self) -> String {
+    async fn subject_str(&self) -> String {
         match self.loaded {
             Loaded::Message { ref chat } => {
                 if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
                     self.context
                         .stock_str(StockMessage::AcSetupMsgSubject)
+                        .await
                         .into_owned()
                 } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
                     let re = if self.in_reply_to.is_empty() {
@@ -317,18 +345,54 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    let raw = message::get_summarytext_by_raw(
-                        self.msg.viewtype,
-                        self.msg.text.as_ref(),
-                        &self.msg.param,
-                        32,
-                        self.context,
-                    );
-                    let raw_subject = raw.lines().next().unwrap_or_default();
-                    format!("Chat: {}", raw_subject)
+                    match chat.param.get(Param::LastSubject) {
+                        Some(last_subject) => {
+                            let subject_start = if last_subject.starts_with("Chat:") {
+                                0
+                            } else {
+                                // "Antw:" is the longest abbreviation in
+                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+                                // so look at the first _5_ characters:
+                                match last_subject.chars().take(5).position(|c| c == ':') {
+                                    Some(prefix_end) => prefix_end + 1,
+                                    None => 0,
+                                }
+                            };
+                            format!(
+                                "Re: {}",
+                                last_subject
+                                    .chars()
+                                    .skip(subject_start)
+                                    .collect::<String>()
+                                    .trim()
+                            )
+                        }
+                        None => {
+                            let self_name = match self.context.get_config(Config::Displayname).await
+                            {
+                                Some(name) => name,
+                                None => self
+                                    .context
+                                    .get_config(Config::Addr)
+                                    .await
+                                    .unwrap_or_default(),
+                            };
+
+                            self.context
+                                .stock_string_repl_str(
+                                    StockMessage::SubjectForNewContact,
+                                    self_name,
+                                )
+                                .await
+                        }
+                    }
                 }
             }
-            Loaded::MDN { .. } => self.context.stock_str(StockMessage::ReadRcpt).into_owned(),
+            Loaded::MDN { .. } => self
+                .context
+                .stock_str(StockMessage::ReadRcpt)
+                .await
+                .into_owned(),
         }
     }
 
@@ -339,7 +403,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .collect()
     }
 
-    pub fn render(mut self) -> Result<RenderedEmail, Error> {
+    pub async fn render(mut self) -> Result<RenderedEmail, Error> {
         // Headers that are encrypted
         // - Chat-*, except Chat-Version
         // - Secure-Join*
@@ -370,6 +434,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
+        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -422,17 +488,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let min_verified = self.min_verified();
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
-        let subject_str = self.subject_str();
+        let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
-        let mut encrypt_helper = EncryptHelper::new(self.context)?;
+        let encrypt_helper = EncryptHelper::new(self.context).await?;
 
         let subject = encode_words(&subject_str);
 
         let mut message = match self.loaded {
             Loaded::Message { .. } => {
-                self.render_message(&mut protected_headers, &mut unprotected_headers, &grpimage)?
+                self.render_message(&mut protected_headers, &mut unprotected_headers, &grpimage)
+                    .await?
             }
-            Loaded::MDN { .. } => self.render_mdn()?,
+            Loaded::MDN { .. } => self.render_mdn().await?,
         };
 
         if force_plaintext != ForcePlaintext::NoAutocryptHeader as i32 {
@@ -443,7 +510,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         protected_headers.push(Header::new("Subject".into(), subject));
 
-        let peerstates = self.peerstates_for_recipients()?;
+        let peerstates = self.peerstates_for_recipients().await?;
         let should_encrypt =
             encrypt_helper.should_encrypt(self.context, e2ee_guaranteed, &peerstates)?;
         let is_encrypted = should_encrypt && force_plaintext == 0;
@@ -471,7 +538,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         let outer_message = if is_encrypted {
             // Add gossip headers in chats with multiple recipients
-            if peerstates.len() > 1 && self.should_do_gossip() {
+            if peerstates.len() > 1 && self.should_do_gossip().await {
                 for peerstate in peerstates.iter().filter_map(|(state, _)| state.as_ref()) {
                     if peerstate.peek_key(min_verified).is_some() {
                         if let Some(header) = peerstate.render_gossip_header(min_verified) {
@@ -519,8 +586,9 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 println!("{}", raw_message);
             }
 
-            let encrypted =
-                encrypt_helper.encrypt(self.context, min_verified, message, &peerstates)?;
+            let encrypted = encrypt_helper
+                .encrypt(self.context, min_verified, message, peerstates)
+                .await?;
 
             outer_message = outer_message
                 .child(
@@ -592,9 +660,9 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         Some(part)
     }
 
-    fn get_location_kml_part(&mut self) -> Result<PartBuilder, Error> {
+    async fn get_location_kml_part(&mut self) -> Result<PartBuilder, Error> {
         let (kml_content, last_added_location_id) =
-            location::get_kml(self.context, self.msg.chat_id)?;
+            location::get_kml(self.context, self.msg.chat_id).await?;
         let part = PartBuilder::new()
             .content_type(
                 &"application/vnd.google-earth.kml+xml"
@@ -614,7 +682,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     }
 
     #[allow(clippy::cognitive_complexity)]
-    fn render_message(
+    async fn render_message(
         &mut self,
         protected_headers: &mut Vec<Header>,
         unprotected_headers: &mut Vec<Header>,
@@ -709,6 +777,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 placeholdertext = Some(
                     self.context
                         .stock_str(StockMessage::AcSetupMsgBody)
+                        .await
                         .to_string(),
                 );
             }
@@ -755,7 +824,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             meta.viewtype = Viewtype::Image;
             meta.param.set(Param::File, grpimage);
 
-            let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image")?;
+            let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
             meta_part = Some(mail);
             protected_headers.push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
         }
@@ -826,13 +895,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         // add attachment part
         if chat::msgtype_has_file(self.msg.viewtype) {
-            if !is_file_size_okay(context, &self.msg) {
+            if !is_file_size_okay(context, &self.msg).await {
                 bail!(
                     "Message exceeds the recommended {} MB.",
                     RECOMMENDED_FILE_SIZE / 1_000_000,
                 );
             } else {
-                let (file_part, _) = build_body_file(context, &self.msg, "")?;
+                let (file_part, _) = build_body_file(context, &self.msg, "").await?;
                 parts.push(file_part);
             }
         }
@@ -845,8 +914,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             parts.push(msg_kml_part);
         }
 
-        if location::is_sending_locations_to_chat(context, self.msg.chat_id) {
-            match self.get_location_kml_part() {
+        if location::is_sending_locations_to_chat(context, self.msg.chat_id).await {
+            match self.get_location_kml_part().await {
                 Ok(part) => parts.push(part),
                 Err(err) => {
                     warn!(context, "mimefactory: could not send location: {}", err);
@@ -855,7 +924,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
 
         if self.attach_selfavatar {
-            match context.get_config(Config::Selfavatar) {
+            match context.get_config(Config::Selfavatar).await {
                 Some(path) => match build_selfavatar_file(context, &path) {
                     Ok((part, filename)) => {
                         parts.push(part);
@@ -882,7 +951,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     }
 
     /// Render an MDN
-    fn render_mdn(&mut self) -> Result<PartBuilder, Error> {
+    async fn render_mdn(&mut self) -> Result<PartBuilder, Error> {
         // RFC 6522, this also requires the `report-type` parameter which is equal
         // to the MIME subtype of the second body part of the multipart/report
         //
@@ -917,13 +986,15 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         {
             self.context
                 .stock_str(StockMessage::EncryptedMsg)
+                .await
                 .into_owned()
         } else {
-            self.msg.get_summarytext(self.context, 32)
+            self.msg.get_summarytext(self.context, 32).await
         };
         let p2 = self
             .context
-            .stock_string_repl_str(StockMessage::ReadRcptMailBody, p1);
+            .stock_string_repl_str(StockMessage::ReadRcptMailBody, p1)
+            .await;
         let message_text = format!("{}\r\n", p2);
         message = message.child(
             PartBuilder::new()
@@ -980,14 +1051,15 @@ fn wrapped_base64_encode(buf: &[u8]) -> String {
         .join("\r\n")
 }
 
-fn build_body_file(
+async fn build_body_file(
     context: &Context,
     msg: &Message,
     base_name: &str,
 ) -> Result<(PartBuilder, String), Error> {
     let blob = msg
         .param
-        .get_blob(Param::File, context, true)?
+        .get_blob(Param::File, context, true)
+        .await?
         .ok_or_else(|| format_err!("msg has no filename"))?;
     let suffix = blob.suffix().unwrap_or("dat");
 
@@ -1083,10 +1155,10 @@ fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool
         .any(|(_, cur)| cur.to_lowercase() == addr_lc)
 }
 
-fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
+async fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
     match msg.param.get_path(Param::File, context).unwrap_or(None) {
         Some(path) => {
-            let bytes = dc_get_filebytes(context, &path);
+            let bytes = dc_get_filebytes(context, &path).await;
             bytes <= UPPER_LIMIT_FILE_SIZE
         }
         None => false,
@@ -1129,6 +1201,10 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chatlist::Chatlist;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::mimeparser::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1136,6 +1212,9 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
+        assert!(!display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1147,6 +1226,25 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
+    #[test]
+    fn test_render_email_address_noescape() {
+        let display_name = "a space";
+        let addr = "x@y.org";
+
+        assert!(display_name.is_ascii());
+        assert!(display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+        let s = format!(
+            "{}",
+            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
+        );
+
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        assert_eq!(s, "a space <x@y.org>");
+    }
+
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1189,4 +1287,193 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
+
+    #[async_std::test]
+    async fn test_subject() {
+        // 1.: Receive a mail from an MUA or Delta Chat
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Infos: 42"
+        );
+
+        // 2. Receive a message from Delta Chat when we did not send any messages before
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        // 3. Send the first message to a new contact
+        let t = TestContext::new_alice().await;
+
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::Displayname, Some("Alice"))
+            .await
+            .unwrap();
+        assert_eq!(first_subject_str(t).await, "Message from Alice");
+
+        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+    }
+
+    async fn first_subject_str(t: TestContext) -> String {
+        let contact_id =
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.com", Origin::ManuallyCreated)
+                .await
+                .unwrap()
+                .0;
+
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+
+        mf.subject_str().await
+    }
+
+    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+        let t = TestContext::new_alice().await;
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        mf.subject_str().await
+    }
+
+    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+        context
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(context, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        new_msg
+    }
+
+    #[async_std::test]
+    // This test could still be extended
+    async fn test_render_reply() {
+        let t = TestContext::new_alice().await;
+        let context = &t.ctx;
+
+        let msg = incoming_msg_to_reply_msg(
+            b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            context,
+        )
+        .await;
+
+        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+
+        let recipients = mimefactory.recipients();
+        assert_eq!(recipients, vec!["charlie@example.com"]);
+
+        let rendered_msg = mimefactory.render().await.unwrap();
+
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        assert_eq!(
+            mail.headers
+                .iter()
+                .find(|h| h.get_key() == "MIME-Version")
+                .unwrap()
+                .get_value(),
+            "1.0"
+        );
+
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+            .await
+            .unwrap();
+    }
 }

src/oauth2.rs

@@ -1,12 +1,16 @@
 //! OAuth 2 module
 
+use regex::Regex;
 use std::collections::HashMap;
 
+use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::provider;
+use crate::provider::Oauth2Authorizer;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
@@ -15,6 +19,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
+    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -24,8 +29,11 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
+    mx_pattern: None,
 };
 
+const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -33,6 +41,7 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
+    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -48,12 +57,12 @@ struct Response {
     scope: Option<String>,
 }
 
-pub fn dc_get_oauth2_url(
+pub async fn dc_get_oauth2_url(
     context: &Context,
     addr: impl AsRef<str>,
     redirect_uri: impl AsRef<str>,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
             .set_raw_config(
@@ -61,6 +70,7 @@ pub fn dc_get_oauth2_url(
                 "oauth2_pending_redirect_uri",
                 Some(redirect_uri.as_ref()),
             )
+            .await
             .is_err()
         {
             return None;
@@ -74,21 +84,18 @@ pub fn dc_get_oauth2_url(
     }
 }
 
-// The following function may block due http-requests;
-// must not be called from the main thread or by the ui!
-pub fn dc_get_oauth2_access_token(
+pub async fn dc_get_oauth2_access_token(
     context: &Context,
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
     regenerate: bool,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
-        let lock = context.oauth2_critical.clone();
-        let _l = lock.lock().unwrap();
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
+        let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
-        if !regenerate && !is_expired(context) {
-            let access_token = context.sql.get_raw_config(context, "oauth2_access_token");
+        if !regenerate && !is_expired(context).await {
+            let access_token = context.sql.get_raw_config("oauth2_access_token").await;
             if access_token.is_some() {
                 // success
                 return access_token;
@@ -96,10 +103,11 @@ pub fn dc_get_oauth2_access_token(
         }
 
         // generate new token: build & call auth url
-        let refresh_token = context.sql.get_raw_config(context, "oauth2_refresh_token");
+        let refresh_token = context.sql.get_raw_config("oauth2_refresh_token").await;
         let refresh_token_for = context
             .sql
-            .get_raw_config(context, "oauth2_refresh_token_for")
+            .get_raw_config("oauth2_refresh_token_for")
+            .await
             .unwrap_or_else(|| "unset".into());
 
         let (redirect_uri, token_url, update_redirect_uri_on_success) =
@@ -108,7 +116,8 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_raw_config(context, "oauth2_pending_redirect_uri")
+                        .get_raw_config("oauth2_pending_redirect_uri")
+                        .await
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.init_token,
                     true,
@@ -121,7 +130,8 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_raw_config(context, "oauth2_redirect_uri")
+                        .get_raw_config("oauth2_redirect_uri")
+                        .await
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.refresh_token,
                     false,
@@ -154,10 +164,7 @@ pub fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = reqwest::blocking::Client::new()
-            .post(post_url)
-            .form(&post_param)
-            .send();
+        let response = surf::post(post_url).body_form(&post_param);
         if response.is_err() {
             warn!(
                 context,
@@ -165,19 +172,8 @@ pub fn dc_get_oauth2_access_token(
             );
             return None;
         }
-        let response = response.unwrap();
-        if !response.status().is_success() {
-            warn!(
-                context,
-                "Unsuccessful response when calling OAuth2 at {}: {:?}",
-                token_url,
-                response.status()
-            );
-            return None;
-        }
 
-        // generate new token: parse returned json
-        let parsed: reqwest::Result<Response> = response.json();
+        let parsed: Result<Response, _> = response.unwrap().recv_json().await;
         if parsed.is_err() {
             warn!(
                 context,
@@ -185,7 +181,6 @@ pub fn dc_get_oauth2_access_token(
             );
             return None;
         }
-        println!("response: {:?}", &parsed);
 
         // update refresh_token if given, typically on the first round, but we update it later as well.
         let response = parsed.unwrap();
@@ -193,10 +188,12 @@ pub fn dc_get_oauth2_access_token(
             context
                 .sql
                 .set_raw_config(context, "oauth2_refresh_token", Some(token))
+                .await
                 .ok();
             context
                 .sql
                 .set_raw_config(context, "oauth2_refresh_token_for", Some(code.as_ref()))
+                .await
                 .ok();
         }
 
@@ -206,6 +203,7 @@ pub fn dc_get_oauth2_access_token(
             context
                 .sql
                 .set_raw_config(context, "oauth2_access_token", Some(token))
+                .await
                 .ok();
             let expires_in = response
                 .expires_in
@@ -215,18 +213,22 @@ pub fn dc_get_oauth2_access_token(
             context
                 .sql
                 .set_raw_config_int64(context, "oauth2_timestamp_expires", expires_in)
+                .await
                 .ok();
 
             if update_redirect_uri_on_success {
                 context
                     .sql
                     .set_raw_config(context, "oauth2_redirect_uri", Some(redirect_uri.as_ref()))
+                    .await
                     .ok();
             }
         } else {
             warn!(context, "Failed to find OAuth2 access token");
         }
 
+        drop(lock);
+
         response.access_token
     } else {
         warn!(context, "Internal OAuth2 error: 2");
@@ -235,22 +237,23 @@ pub fn dc_get_oauth2_access_token(
     }
 }
 
-pub fn dc_get_oauth2_addr(
+pub async fn dc_get_oauth2_addr(
     context: &Context,
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    let oauth2 = Oauth2::from_address(addr.as_ref()).await?;
     oauth2.get_userinfo?;
 
     if let Some(access_token) =
-        dc_get_oauth2_access_token(context, addr.as_ref(), code.as_ref(), false)
+        dc_get_oauth2_access_token(context, addr.as_ref(), code.as_ref(), false).await
     {
-        let addr_out = oauth2.get_addr(context, access_token);
+        let addr_out = oauth2.get_addr(context, access_token).await;
         if addr_out.is_none() {
             // regenerate
-            if let Some(access_token) = dc_get_oauth2_access_token(context, addr, code, true) {
-                oauth2.get_addr(context, access_token)
+            if let Some(access_token) = dc_get_oauth2_access_token(context, addr, code, true).await
+            {
+                oauth2.get_addr(context, access_token).await
             } else {
                 None
             }
@@ -263,24 +266,57 @@ pub fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    fn from_address(addr: impl AsRef<str>) -> Option<Self> {
+    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
         let addr_normalized = normalize_addr(addr.as_ref());
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            match domain {
-                "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
-                | "narod.ru" => Some(OAUTH2_YANDEX),
-                _ => None,
+            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
+                match &provider.oauth2_authorizer {
+                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
+                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
+                    None => None, // provider known to not support oauth2, no mx-lookup required
+                }
+            } else {
+                Oauth2::lookup_mx(domain).await
             }
         } else {
             None
         }
     }
 
-    fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
+    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
+        if let Ok(resolver) = resolver(
+            config::ResolverConfig::default(),
+            config::ResolverOpts::default(),
+        )
+        .await
+        {
+            for provider in OAUTH2_PROVIDERS.iter() {
+                if let Some(pattern) = provider.mx_pattern {
+                    let re = Regex::new(pattern).unwrap();
+
+                    let mut fqdn: String = String::from(domain.as_ref());
+                    if !fqdn.ends_with('.') {
+                        fqdn.push_str(".");
+                    }
+
+                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
+                        for rr in res.iter() {
+                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
+                                return Some(provider.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
+    async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
         let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
 
@@ -291,50 +327,35 @@ impl Oauth2 {
         //   "verified_email": true,
         //   "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg"
         // }
-        let response = reqwest::blocking::Client::new().get(&userinfo_url).send();
+        let response: Result<HashMap<String, serde_json::Value>, surf::Error> =
+            surf::get(userinfo_url).recv_json().await;
         if response.is_err() {
             warn!(context, "Error getting userinfo: {:?}", response);
             return None;
         }
-        let response = response.unwrap();
-        if !response.status().is_success() {
-            warn!(context, "Error getting userinfo: {:?}", response.status());
-            return None;
-        }
 
-        let parsed: reqwest::Result<HashMap<String, serde_json::Value>> = response.json();
-        if parsed.is_err() {
-            warn!(
-                context,
-                "Failed to parse userinfo JSON response: {:?}", parsed
-            );
-            return None;
-        }
-        if let Ok(response) = parsed {
-            // CAVE: serde_json::Value.as_str() removes the quotes of json-strings
-            // but serde_json::Value.to_string() does not!
-            if let Some(addr) = response.get("email") {
-                if let Some(s) = addr.as_str() {
-                    Some(s.to_string())
-                } else {
-                    warn!(context, "E-mail in userinfo is not a string: {}", addr);
-                    None
-                }
+        let parsed = response.unwrap();
+        // CAVE: serde_json::Value.as_str() removes the quotes of json-strings
+        // but serde_json::Value.to_string() does not!
+        if let Some(addr) = parsed.get("email") {
+            if let Some(s) = addr.as_str() {
+                Some(s.to_string())
             } else {
-                warn!(context, "E-mail missing in userinfo.");
+                warn!(context, "E-mail in userinfo is not a string: {}", addr);
                 None
             }
         } else {
-            warn!(context, "Failed to parse userinfo.");
+            warn!(context, "E-mail missing in userinfo.");
             None
         }
     }
 }
 
-fn is_expired(context: &Context) -> bool {
+async fn is_expired(context: &Context) -> bool {
     let expire_timestamp = context
         .sql
-        .get_raw_config_int64(context, "oauth2_timestamp_expires")
+        .get_raw_config_int64("oauth2_timestamp_expires")
+        .await
         .unwrap_or_default();
 
     if expire_timestamp <= 0 {
@@ -377,48 +398,62 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_oauth_from_address() {
-        assert_eq!(Oauth2::from_address("hello@gmail.com"), Some(OAUTH2_GMAIL));
+    #[async_std::test]
+    async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com"),
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com"),
+            Oauth2::from_address("hello@googlemail.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.com").await,
+            Some(OAUTH2_YANDEX)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address("hello@yandex.ru"), Some(OAUTH2_YANDEX));
 
-        assert_eq!(Oauth2::from_address("hello@web.de"), None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
     }
 
-    #[test]
-    fn test_dc_get_oauth2_addr() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_oauth_from_mx() {
+        assert_eq!(
+            Oauth2::from_address("hello@google.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+    }
+
+    #[async_std::test]
+    async fn test_dc_get_oauth2_addr() {
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
-        let res = dc_get_oauth2_addr(&ctx.ctx, addr, code);
+        let res = dc_get_oauth2_addr(&ctx.ctx, addr, code).await;
         // this should fail as it is an invalid password
         assert_eq!(res, None);
     }
 
-    #[test]
-    fn test_dc_get_oauth2_url() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_dc_get_oauth2_url() {
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
-        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri);
+        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
 
         assert_eq!(res, Some("https://accounts.google.com/o/oauth2/auth?client_id=959970109878%2D4mvtgf6feshskf7695nfln6002mom908%2Eapps%2Egoogleusercontent%2Ecom&redirect_uri=chat%2Edelta%3A%2Fcom%2Eb44t%2Emessenger&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline".into()));
     }
 
-    #[test]
-    fn test_dc_get_oauth2_token() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_dc_get_oauth2_token() {
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
-        let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false);
+        let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false).await;
         // this should fail as it is an invalid password
         assert_eq!(res, None);
     }

src/param.rs

@@ -1,8 +1,8 @@
 use std::collections::BTreeMap;
 use std::fmt;
-use std::path::PathBuf;
 use std::str;
 
+use async_std::path::PathBuf;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
@@ -65,9 +65,6 @@ pub enum Param {
     /// For Messages
     Arg4 = b'H',
 
-    /// For Messages
-    Error = b'L',
-
     /// For Messages
     AttachGroupImage = b'A',
 
@@ -103,6 +100,9 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
+    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    LastSubject = b't',
+
     /// For Chats
     Devicetalk = b'D',
 
@@ -275,7 +275,8 @@ impl Params {
     /// created without copying if the path already referes to a valid
     /// blob.  If so a [BlobObject] will be returned regardless of the
     /// `create` argument.
-    pub fn get_blob<'a>(
+    #[allow(clippy::needless_lifetimes)]
+    pub async fn get_blob<'a>(
         &self,
         key: Param,
         context: &'a Context,
@@ -288,7 +289,7 @@ impl Params {
         let file = ParamsFile::from_param(context, val)?;
         let blob = match file {
             ParamsFile::FsPath(path) => match create {
-                true => BlobObject::new_from_path(context, path)?,
+                true => BlobObject::new_from_path(context, path).await?,
                 false => BlobObject::from_path(context, path)?,
             },
             ParamsFile::Blob(blob) => blob,
@@ -362,8 +363,8 @@ impl<'a> ParamsFile<'a> {
 mod tests {
     use super::*;
 
-    use std::fs;
-    use std::path::Path;
+    use async_std::fs;
+    use async_std::path::Path;
 
     use crate::test_utils::*;
 
@@ -411,9 +412,9 @@ mod tests {
         assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
     }
 
-    #[test]
-    fn test_params_file_fs_path() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_params_file_fs_path() {
+        let t = TestContext::new().await;
         if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
@@ -421,9 +422,9 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_params_file_blob() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_params_file_blob() {
+        let t = TestContext::new().await;
         if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
@@ -432,28 +433,33 @@ mod tests {
     }
 
     // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
-    #[test]
-    fn test_params_get_fileparam() {
-        let t = dummy_context();
+    #[async_std::test]
+    async fn test_params_get_fileparam() {
+        let t = TestContext::new().await;
         let fname = t.dir.path().join("foo");
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());
 
         let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
-        assert_eq!(file, ParamsFile::FsPath(fname.clone()));
+        assert_eq!(file, ParamsFile::FsPath(fname.clone().into()));
 
-        let path = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        let path: PathBuf = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        let fname: PathBuf = fname.into();
         assert_eq!(path, fname);
 
         // Blob does not exist yet, expect BlobError.
-        let err = p.get_blob(Param::File, &t.ctx, false).unwrap_err();
+        let err = p.get_blob(Param::File, &t.ctx, false).await.unwrap_err();
         match err {
             BlobError::WrongBlobdir { .. } => (),
             _ => panic!("wrong error type/variant: {:?}", err),
         }
 
-        fs::write(fname, b"boo").unwrap();
-        let blob = p.get_blob(Param::File, &t.ctx, true).unwrap().unwrap();
+        fs::write(fname, b"boo").await.unwrap();
+        let blob = p
+            .get_blob(Param::File, &t.ctx, true)
+            .await
+            .unwrap()
+            .unwrap();
         assert_eq!(
             blob,
             BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
@@ -462,7 +468,11 @@ mod tests {
         // Blob in blobdir, expect blob.
         let bar = t.ctx.get_blobdir().join("bar");
         p.set(Param::File, bar.to_str().unwrap());
-        let blob = p.get_blob(Param::File, &t.ctx, false).unwrap().unwrap();
+        let blob = p
+            .get_blob(Param::File, &t.ctx, false)
+            .await
+            .unwrap()
+            .unwrap();
         assert_eq!(
             blob,
             BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
@@ -471,6 +481,10 @@ mod tests {
         p.remove(Param::File);
         assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
         assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p.get_blob(Param::File, &t.ctx, false).unwrap().is_none());
+        assert!(p
+            .get_blob(Param::File, &t.ctx, false)
+            .await
+            .unwrap()
+            .is_none());
     }
 }

src/peerstate.rs

@@ -1,15 +1,13 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
-use std::collections::HashSet;
-use std::convert::TryFrom;
-use std::fmt;
 
-use num_traits::FromPrimitive;
+use std::collections::HashSet;
+
+use anyhow::Result;
 
 use crate::aheader::*;
-use crate::constants::*;
 use crate::context::Context;
-use crate::key::{Key, SignedPublicKey};
-use crate::sql::{self, Sql};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
+use crate::sql::Sql;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -26,60 +24,66 @@ pub enum PeerstateVerifiedStatus {
 }
 
 /// Peerstate represents the state of an Autocrypt peer.
-pub struct Peerstate<'a> {
-    pub context: &'a Context,
+#[derive(Debug, PartialEq, Eq)]
+pub struct Peerstate {
     pub addr: String,
     pub last_seen: i64,
     pub last_seen_autocrypt: i64,
     pub prefer_encrypt: EncryptPreference,
-    pub public_key: Option<Key>,
-    pub public_key_fingerprint: Option<String>,
-    pub gossip_key: Option<Key>,
+    pub public_key: Option<SignedPublicKey>,
+    pub public_key_fingerprint: Option<Fingerprint>,
+    pub gossip_key: Option<SignedPublicKey>,
     pub gossip_timestamp: i64,
-    pub gossip_key_fingerprint: Option<String>,
-    pub verified_key: Option<Key>,
-    pub verified_key_fingerprint: Option<String>,
+    pub gossip_key_fingerprint: Option<Fingerprint>,
+    pub verified_key: Option<SignedPublicKey>,
+    pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
     pub degrade_event: Option<DegradeEvent>,
 }
 
-impl<'a> PartialEq for Peerstate<'a> {
-    fn eq(&self, other: &Peerstate) -> bool {
-        self.addr == other.addr
-            && self.last_seen == other.last_seen
-            && self.last_seen_autocrypt == other.last_seen_autocrypt
-            && self.prefer_encrypt == other.prefer_encrypt
-            && self.public_key == other.public_key
-            && self.public_key_fingerprint == other.public_key_fingerprint
-            && self.gossip_key == other.gossip_key
-            && self.gossip_timestamp == other.gossip_timestamp
-            && self.gossip_key_fingerprint == other.gossip_key_fingerprint
-            && self.verified_key == other.verified_key
-            && self.verified_key_fingerprint == other.verified_key_fingerprint
-            && self.to_save == other.to_save
-            && self.degrade_event == other.degrade_event
-    }
-}
+impl<'a> sqlx::FromRow<'a, sqlx::sqlite::SqliteRow> for Peerstate {
+    fn from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Self, sqlx::Error> {
+        use sqlx::Row;
 
-impl<'a> Eq for Peerstate<'a> {}
+        let mut res = Self::new(row.try_get("addr")?);
 
-impl<'a> fmt::Debug for Peerstate<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.debug_struct("Peerstate")
-            .field("addr", &self.addr)
-            .field("last_seen", &self.last_seen)
-            .field("last_seen_autocrypt", &self.last_seen_autocrypt)
-            .field("prefer_encrypt", &self.prefer_encrypt)
-            .field("public_key", &self.public_key)
-            .field("public_key_fingerprint", &self.public_key_fingerprint)
-            .field("gossip_key", &self.gossip_key)
-            .field("gossip_timestamp", &self.gossip_timestamp)
-            .field("gossip_key_fingerprint", &self.gossip_key_fingerprint)
-            .field("verified_key", &self.verified_key)
-            .field("verified_key_fingerprint", &self.verified_key_fingerprint)
-            .field("to_save", &self.to_save)
-            .field("degrade_event", &self.degrade_event)
-            .finish()
+        res.last_seen = row.try_get_unchecked("last_seen")?;
+        res.last_seen_autocrypt = row.try_get_unchecked("last_seen_autocrypt")?;
+        res.prefer_encrypt = row.try_get("prefer_encrypted")?;
+        res.gossip_timestamp = row.try_get_unchecked("gossip_timestamp")?;
+
+        res.public_key_fingerprint = row
+            .try_get::<Option<String>, _>("public_key_fingerprint")?
+            .map(|fp| fp.parse::<Fingerprint>())
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.gossip_key_fingerprint = row
+            .try_get::<Option<String>, _>("gossip_key_fingerprint")?
+            .map(|fp| fp.parse::<Fingerprint>())
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.verified_key_fingerprint = row
+            .try_get::<Option<String>, _>("verified_key_fingerprint")?
+            .map(|fp| fp.parse::<Fingerprint>())
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.public_key = row
+            .try_get::<Option<&[u8]>, _>("public_key")?
+            .map(|blob| SignedPublicKey::from_slice(blob))
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.gossip_key = row
+            .try_get::<Option<&[u8]>, _>("gossip_key")?
+            .map(|blob| SignedPublicKey::from_slice(blob))
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+        res.verified_key = row
+            .try_get::<Option<&[u8]>, _>("verified_key")?
+            .map(|blob| SignedPublicKey::from_slice(blob))
+            .transpose()
+            .map_err(|err| sqlx::Error::Decode(Box::new(err)))?;
+
+        Ok(res)
     }
 }
 
@@ -100,10 +104,9 @@ pub enum DegradeEvent {
     FingerprintChanged = 0x02,
 }
 
-impl<'a> Peerstate<'a> {
-    pub fn new(context: &'a Context, addr: String) -> Self {
+impl Peerstate {
+    pub fn new(addr: String) -> Self {
         Peerstate {
-            context,
             addr,
             last_seen: 0,
             last_seen_autocrypt: 0,
@@ -120,114 +123,77 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, header.addr.clone());
+    pub fn from_header(header: &Aheader, message_time: i64) -> Self {
+        let mut res = Self::new(header.addr.clone());
 
         res.last_seen = message_time;
         res.last_seen_autocrypt = message_time;
         res.to_save = Some(ToSave::All);
         res.prefer_encrypt = header.prefer_encrypt;
-        res.public_key = Some(Key::from(header.public_key.clone()));
+        res.public_key = Some(header.public_key.clone());
         res.recalc_fingerprint();
 
         res
     }
 
-    pub fn from_gossip(context: &'a Context, gossip_header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, gossip_header.addr.clone());
+    pub fn from_gossip(gossip_header: &Aheader, message_time: i64) -> Self {
+        let mut res = Self::new(gossip_header.addr.clone());
 
         res.gossip_timestamp = message_time;
         res.to_save = Some(ToSave::All);
-        res.gossip_key = Some(Key::from(gossip_header.public_key.clone()));
+        res.gossip_key = Some(gossip_header.public_key.clone());
         res.recalc_fingerprint();
 
         res
     }
 
-    pub fn from_addr(context: &'a Context, _sql: &Sql, addr: &str) -> Option<Self> {
-        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, verified_key, verified_key_fingerprint FROM acpeerstates  WHERE addr=? COLLATE NOCASE;";
-        Self::from_stmt(context, query, &[addr])
+    pub async fn from_addr(context: &Context, addr: &str) -> Result<Peerstate> {
+        let query = r#"
+SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, 
+       gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, 
+       verified_key, verified_key_fingerprint 
+  FROM acpeerstates  
+  WHERE addr=? COLLATE NOCASE;
+"#;
+        Self::from_stmt(context, query, paramsx![addr]).await
     }
 
-    pub fn from_fingerprint(context: &'a Context, _sql: &Sql, fingerprint: &str) -> Option<Self> {
-        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
-                     gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint \
-                     FROM acpeerstates  \
-                     WHERE public_key_fingerprint=? COLLATE NOCASE \
-                     OR gossip_key_fingerprint=? COLLATE NOCASE  \
-                     ORDER BY public_key_fingerprint=? DESC;";
+    pub async fn from_fingerprint(
+        context: &Context,
+        fingerprint: &Fingerprint,
+    ) -> Result<Peerstate> {
+        let query = r#"
+SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key,
+       gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint,
+       verified_key, verified_key_fingerprint
+  FROM acpeerstates
+  WHERE public_key_fingerprint=? COLLATE NOCASE
+    OR gossip_key_fingerprint=? COLLATE NOCASE
+  ORDER BY public_key_fingerprint=? DESC;
+"#;
 
+        let fingerprint = fingerprint.hex();
         Self::from_stmt(
             context,
             query,
-            params![fingerprint, fingerprint, fingerprint],
+            paramsx![&fingerprint, &fingerprint, &fingerprint],
         )
+        .await
     }
 
-    fn from_stmt<P>(context: &'a Context, query: &str, params: P) -> Option<Self>
-    where
-        P: IntoIterator,
-        P::Item: rusqlite::ToSql,
-    {
-        context
-            .sql
-            .query_row(query, params, |row| {
-                /* all the above queries start with this: SELECT
-                addr, last_seen, last_seen_autocrypt, prefer_encrypted,
-                public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
-                gossip_key_fingerprint, verified_key, verified_key_fingerprint
-                */
-                let mut res = Self::new(context, row.get(0)?);
+    async fn from_stmt<'a, P: sqlx::IntoArguments<'a, sqlx::sqlite::Sqlite> + 'a>(
+        context: &Context,
+        query: &'a str,
+        params: P,
+    ) -> Result<Peerstate> {
+        /* all the above queries start with this: SELECT
+        addr, last_seen, last_seen_autocrypt, prefer_encrypted,
+        public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
+        gossip_key_fingerprint, verified_key, verified_key_fingerprint
+        */
+        let peerstate = context.sql.query_row(query, params).await?;
 
-                res.last_seen = row.get(1)?;
-                res.last_seen_autocrypt = row.get(2)?;
-                res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
-                res.gossip_timestamp = row.get(5)?;
-
-                res.public_key_fingerprint = row.get(7)?;
-                if res
-                    .public_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.public_key_fingerprint = None;
-                }
-                res.gossip_key_fingerprint = row.get(8)?;
-                if res
-                    .gossip_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.gossip_key_fingerprint = None;
-                }
-                res.verified_key_fingerprint = row.get(10)?;
-                if res
-                    .verified_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.verified_key_fingerprint = None;
-                }
-                res.public_key = row
-                    .get(4)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
-                res.gossip_key = row
-                    .get(6)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
-                res.verified_key = row
-                    .get(9)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
-
-                Ok(res)
-            })
-            .ok()
+        Ok(peerstate)
     }
 
     pub fn recalc_fingerprint(&mut self) {
@@ -294,8 +260,8 @@ impl<'a> Peerstate<'a> {
                 self.to_save = Some(ToSave::All)
             }
 
-            if self.public_key.as_ref() != Some(&Key::from(header.public_key.clone())) {
-                self.public_key = Some(Key::from(header.public_key.clone()));
+            if self.public_key.as_ref() != Some(&header.public_key) {
+                self.public_key = Some(header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All);
             }
@@ -310,9 +276,8 @@ impl<'a> Peerstate<'a> {
         if message_time > self.gossip_timestamp {
             self.gossip_timestamp = message_time;
             self.to_save = Some(ToSave::Timestamps);
-            let hdr_key = Key::from(gossip_header.public_key.clone());
-            if self.gossip_key.as_ref() != Some(&hdr_key) {
-                self.gossip_key = Some(hdr_key);
+            if self.gossip_key.as_ref() != Some(&gossip_header.public_key) {
+                self.gossip_key = Some(gossip_header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All)
             }
@@ -340,11 +305,9 @@ impl<'a> Peerstate<'a> {
 
     pub fn render_gossip_header(&self, min_verified: PeerstateVerifiedStatus) -> Option<String> {
         if let Some(key) = self.peek_key(min_verified) {
-            // TODO: avoid cloning
-            let public_key = SignedPublicKey::try_from(key.clone()).ok()?;
             let header = Aheader::new(
                 self.addr.clone(),
-                public_key,
+                key.clone(), // TODO: avoid cloning
                 // Autocrypt 1.1.0 specification says that
                 // `prefer-encrypt` attribute SHOULD NOT be included,
                 // but we include it anyway to propagate encryption
@@ -361,7 +324,16 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&Key> {
+    pub fn take_key(mut self, min_verified: PeerstateVerifiedStatus) -> Option<SignedPublicKey> {
+        match min_verified {
+            PeerstateVerifiedStatus::BidirectVerified => self.verified_key.take(),
+            PeerstateVerifiedStatus::Unverified => {
+                self.public_key.take().or_else(|| self.gossip_key.take())
+            }
+        }
+    }
+
+    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&SignedPublicKey> {
         match min_verified {
             PeerstateVerifiedStatus::BidirectVerified => self.verified_key.as_ref(),
             PeerstateVerifiedStatus::Unverified => self
@@ -374,7 +346,7 @@ impl<'a> Peerstate<'a> {
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
         verified: PeerstateVerifiedStatus,
     ) -> bool {
         if verified == PeerstateVerifiedStatus::BidirectVerified {
@@ -409,66 +381,60 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn save_to_db(&self, sql: &Sql, create: bool) -> crate::sql::Result<()> {
+    pub async fn save_to_db(&self, sql: &Sql, create: bool) -> crate::sql::Result<()> {
         if create {
-            sql::execute(
-                self.context,
-                sql,
+            sql.execute(
                 "INSERT INTO acpeerstates (addr) VALUES(?);",
-                params![self.addr],
-            )?;
+                paramsx![&self.addr],
+            )
+            .await?;
         }
 
         if self.to_save == Some(ToSave::All) || create {
-            sql::execute(
-                self.context,
-                sql,
-                "UPDATE acpeerstates \
-                 SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
-                 public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
-                 verified_key=?, verified_key_fingerprint=? \
-                 WHERE addr=?;",
-                params![
+            sql.execute(
+                r#"
+UPDATE acpeerstates 
+  SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?,
+      public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?,
+      verified_key=?, verified_key_fingerprint=?
+  WHERE addr=?;
+"#,
+                paramsx![
                     self.last_seen,
                     self.last_seen_autocrypt,
                     self.prefer_encrypt as i64,
                     self.public_key.as_ref().map(|k| k.to_bytes()),
                     self.gossip_timestamp,
                     self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    &self.public_key_fingerprint,
-                    &self.gossip_key_fingerprint,
+                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    &self.verified_key_fingerprint,
-                    &self.addr,
+                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    &self.addr
                 ],
-            )?;
+            ).await?;
         } else if self.to_save == Some(ToSave::Timestamps) {
-            sql::execute(
-                self.context,
-                sql,
-                "UPDATE acpeerstates SET last_seen=?, last_seen_autocrypt=?, gossip_timestamp=? \
-                 WHERE addr=?;",
-                params![
+            sql.execute(
+                "UPDATE acpeerstates SET last_seen=?, last_seen_autocrypt=?, gossip_timestamp=? WHERE addr=?;",
+                paramsx![
                     self.last_seen,
                     self.last_seen_autocrypt,
                     self.gossip_timestamp,
                     &self.addr
                 ],
-            )?;
+            )
+            .await?;
         }
 
         Ok(())
     }
 
-    pub fn has_verified_key(&self, fingerprints: &HashSet<String>) -> bool {
-        if self.verified_key.is_some() && self.verified_key_fingerprint.is_some() {
-            let vkc = self.verified_key_fingerprint.as_ref().unwrap();
-            if fingerprints.contains(vkc) {
-                return true;
-            }
+    pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
+        if let Some(vkc) = &self.verified_key_fingerprint {
+            fingerprints.contains(vkc) && self.verified_key.is_some()
+        } else {
+            false
         }
-
-        false
     }
 }
 
@@ -477,17 +443,15 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
     use pretty_assertions::assert_eq;
-    use tempfile::TempDir;
 
-    #[test]
-    fn test_peerstate_save_to_db() {
-        let ctx = crate::test_utils::dummy_context();
+    #[async_std::test]
+    async fn test_peerstate_save_to_db() {
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -504,30 +468,30 @@ mod tests {
         };
 
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "failed to save to db"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, addr)
+            .await
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
         assert_eq!(peerstate, peerstate_new);
-        let peerstate_new2 =
-            Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint())
-                .expect("failed to load peerstate from db");
+        let peerstate_new2 = Peerstate::from_fingerprint(&ctx.ctx, &pub_key.fingerprint())
+            .await
+            .expect("failed to load peerstate from db");
         assert_eq!(peerstate, peerstate_new2);
     }
 
-    #[test]
-    fn test_peerstate_double_create() {
-        let ctx = crate::test_utils::dummy_context();
+    #[async_std::test]
+    async fn test_peerstate_double_create() {
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -544,24 +508,23 @@ mod tests {
         };
 
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "failed to save"
         );
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "double-call with create failed"
         );
     }
 
-    #[test]
-    fn test_peerstate_with_empty_gossip_key_save_to_db() {
-        let ctx = crate::test_utils::dummy_context();
+    #[async_std::test]
+    async fn test_peerstate_with_empty_gossip_key_save_to_db() {
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -578,22 +541,16 @@ mod tests {
         };
 
         assert!(
-            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(),
             "failed to save"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, addr)
+            .await
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
         assert_eq!(peerstate, peerstate_new);
     }
-
-    // TODO: don't copy this from stress.rs
-    #[allow(dead_code)]
-    struct TestContext {
-        ctx: Context,
-        dir: TempDir,
-    }
 }

src/pgp.rs

@@ -1,7 +1,6 @@
 //! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
 
 use std::collections::{BTreeMap, HashSet};
-use std::convert::TryInto;
 use std::io;
 use std::io::Cursor;
 
@@ -19,8 +18,8 @@ use rand::{thread_rng, CryptoRng, Rng};
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::key::*;
-use crate::keyring::*;
+use crate::key::{DcKey, Fingerprint};
+use crate::keyring::Keyring;
 
 pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
 pub const HEADER_SETUPCODE: &str = "passphrase-begin";
@@ -238,124 +237,142 @@ fn select_pk_for_encryption(key: &SignedPublicKey) -> Option<SignedPublicKeyOrSu
 
 /// Encrypts `plain` text using `public_keys_for_encryption`
 /// and signs it using `private_key_for_signing`.
-pub fn pk_encrypt(
+pub async fn pk_encrypt(
     plain: &[u8],
-    public_keys_for_encryption: &Keyring,
-    private_key_for_signing: Option<&Key>,
+    public_keys_for_encryption: Keyring<SignedPublicKey>,
+    private_key_for_signing: Option<SignedSecretKey>,
 ) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
-    let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            key.as_ref()
-                .try_into()
-                .ok()
-                .and_then(select_pk_for_encryption)
-        })
-        .collect();
-    let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    let mut rng = thread_rng();
+    async_std::task::spawn_blocking(move || {
+        let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
+            .keys()
+            .iter()
+            .filter_map(|key| select_pk_for_encryption(key))
+            .collect();
+        let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    // TODO: measure time
-    let encrypted_msg = if let Some(private_key) = private_key_for_signing {
-        let skey: &SignedSecretKey = private_key
-            .try_into()
-            .map_err(|_| format_err!("Invalid private key"))?;
+        let mut rng = thread_rng();
 
-        lit_msg
-            .sign(skey, || "".into(), Default::default())
-            .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
-            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
-    } else {
-        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
-    };
+        // TODO: measure time
+        let encrypted_msg = if let Some(ref skey) = private_key_for_signing {
+            lit_msg
+                .sign(skey, || "".into(), Default::default())
+                .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
+                .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
+        } else {
+            lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
+        };
 
-    let msg = encrypted_msg?;
-    let encoded_msg = msg.to_armored_string(None)?;
+        let msg = encrypted_msg?;
+        let encoded_msg = msg.to_armored_string(None)?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
+    .await
 }
 
+/// Decrypts the message with keys from the private key keyring.
+///
+/// Receiver private keys are provided in
+/// `private_keys_for_decryption`.
+///
+/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// of all keys from the `public_keys_for_validation` keyring that
+/// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
-pub fn pk_decrypt(
-    ctext: &[u8],
-    private_keys_for_decryption: &Keyring,
-    public_keys_for_validation: &Keyring,
-    ret_signature_fingerprints: Option<&mut HashSet<String>>,
+pub async fn pk_decrypt(
+    ctext: Vec<u8>,
+    private_keys_for_decryption: Keyring<SignedSecretKey>,
+    public_keys_for_validation: Keyring<SignedPublicKey>,
+    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
 ) -> Result<Vec<u8>> {
-    let (msg, _) = Message::from_armor_single(Cursor::new(ctext))?;
-    let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            let k: &Key = &key;
-            k.try_into().ok()
-        })
-        .collect();
+    let msgs = async_std::task::spawn_blocking(move || {
+        let cursor = Cursor::new(ctext);
+        let (msg, _) = Message::from_armor_single(cursor)?;
 
-    let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-    ensure!(!msgs.is_empty(), "No valid messages found");
+        let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption.keys().iter().collect();
 
-    let dec_msg = &msgs[0];
+        let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
+        decryptor.collect::<pgp::errors::Result<Vec<_>>>()
+    })
+    .await?;
 
-    if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-        if !public_keys_for_validation.keys().is_empty() {
-            let pkeys: Vec<&SignedPublicKey> = public_keys_for_validation
-                .keys()
-                .iter()
-                .filter_map(|key| {
-                    let k: &Key = &key;
-                    k.try_into().ok()
+    if let Some(msg) = msgs.into_iter().next() {
+        // get_content() will decompress the message if needed,
+        // but this avoids decompressing it again to check signatures
+        let msg = msg.decompress()?;
+
+        let content = match msg.get_content()? {
+            Some(content) => content,
+            None => bail!("The decrypted message is empty"),
+        };
+
+        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
+            if !public_keys_for_validation.is_empty() {
+                let fingerprints = async_std::task::spawn_blocking(move || {
+                    let pkeys = public_keys_for_validation.keys();
+
+                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                        for pkey in pkeys {
+                            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                                let fp = DcKey::fingerprint(pkey);
+                                fingerprints.push(fp);
+                            }
+                        }
+                    }
+                    fingerprints
                 })
-                .collect();
+                .await;
 
-            for pkey in &pkeys {
-                if dec_msg.verify(&pkey.primary_key).is_ok() {
-                    let fp = hex::encode_upper(pkey.fingerprint());
-                    ret_signature_fingerprints.insert(fp);
-                }
+                ret_signature_fingerprints.extend(fingerprints);
             }
         }
-    }
-
-    match dec_msg.get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
+        Ok(content)
+    } else {
+        bail!("No valid messages found");
     }
 }
 
 /// Symmetric encryption.
-pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
-    let mut rng = thread_rng();
+pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
+    let passphrase = passphrase.to_string();
 
-    let s2k = StringToKey::new_default(&mut rng);
-    let msg =
-        lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase.into())?;
+    async_std::task::spawn_blocking(move || {
+        let mut rng = thread_rng();
+        let s2k = StringToKey::new_default(&mut rng);
+        let msg =
+            lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase)?;
 
-    let encoded_msg = msg.to_armored_string(None)?;
+        let encoded_msg = msg.to_armored_string(None)?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
+    .await
 }
 
 /// Symmetric decryption.
-pub fn symm_decrypt<T: std::io::Read + std::io::Seek>(
+pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     ctext: T,
 ) -> Result<Vec<u8>> {
     let (enc_msg, _) = Message::from_armor_single(ctext)?;
-    let decryptor = enc_msg.decrypt_with_password(|| passphrase.into())?;
 
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-    ensure!(!msgs.is_empty(), "No valid messages found");
+    let passphrase = passphrase.to_string();
+    async_std::task::spawn_blocking(move || {
+        let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
-    match msgs[0].get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
-    }
+        let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
+        ensure!(!msgs.is_empty(), "No valid messages found");
+
+        match msgs[0].get_content()? {
+            Some(content) => Ok(content),
+            None => bail!("Decrypted message is empty"),
+        }
+    })
+    .await
 }
 
 #[cfg(test)]
@@ -408,10 +425,10 @@ mod tests {
 
     /// [Key] objects to use in tests.
     struct TestKeys {
-        alice_secret: Key,
-        alice_public: Key,
-        bob_secret: Key,
-        bob_public: Key,
+        alice_secret: SignedSecretKey,
+        alice_public: SignedPublicKey,
+        bob_secret: SignedSecretKey,
+        bob_public: SignedPublicKey,
     }
 
     impl TestKeys {
@@ -419,10 +436,10 @@ mod tests {
             let alice = alice_keypair();
             let bob = bob_keypair();
             TestKeys {
-                alice_secret: Key::from(alice.secret.clone()),
-                alice_public: Key::from(alice.public.clone()),
-                bob_secret: Key::from(bob.secret.clone()),
-                bob_public: Key::from(bob.public.clone()),
+                alice_secret: alice.secret.clone(),
+                alice_public: alice.public.clone(),
+                bob_secret: bob.secret.clone(),
+                bob_public: bob.public.clone(),
             }
         }
     }
@@ -436,18 +453,18 @@ mod tests {
 
         /// A cyphertext encrypted to Alice & Bob, signed by Alice.
         static ref CTEXT_SIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, Some(&KEYS.alice_secret)).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
         };
 
         /// A cyphertext encrypted to Alice & Bob, not signed.
         static ref CTEXT_UNSIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, None).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
         };
     }
 
@@ -463,110 +480,115 @@ mod tests {
         assert!(CTEXT_UNSIGNED.starts_with("-----BEGIN PGP MESSAGE-----"));
     }
 
-    #[test]
-    fn test_decrypt_singed() {
+    #[async_std::test]
+    async fn test_decrypt_singed() {
         // Check decrypting as Alice
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring: Keyring<SignedSecretKey> = Keyring::new();
+        decrypt_keyring.add(KEYS.alice_secret.clone());
+        let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
 
         // Check decrypting as Bob
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
     }
 
-    #[test]
-    fn test_decrypt_no_sig_check() {
-        let mut keyring = Keyring::default();
-        keyring.add_ref(&KEYS.alice_secret);
-        let empty_keyring = Keyring::default();
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_no_sig_check() {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_secret.clone());
+        let empty_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &keyring,
-            &empty_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            keyring,
+            empty_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_key() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_key() {
         // The validation does not have the public key of the signer.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.bob_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.bob_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_unsigned() {
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let sig_check_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_unsigned() {
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let sig_check_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_UNSIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_UNSIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_sigret() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_sigret() {
         // Check decrypting signed cyphertext without providing the HashSet for signatures.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             None,
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
     }

src/provider/data.rs

@@ -18,6 +18,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // aol.md: aol.com
@@ -28,6 +31,24 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/aol",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // arcor.de.md: arcor.de
+    static ref P_ARCOR_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/arcor-de",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.arcor.de", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.arcor.de", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // autistici.org.md: autistici.org
@@ -40,6 +61,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mail.autistici.org", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -52,16 +76,82 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imaps.bluewin.ch", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // chello.at.md: chello.at
+    static ref P_CHELLO_AT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/chello-at",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "mail.mymagenta.at", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.mymagenta.at", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_COMCAST: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/comcast",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // dismail.de.md: dismail.de
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISMAIL_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dismail-de",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // disroot.md: disroot.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISROOT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/disroot",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
+
+    // dubby.org.md: dubby.org
+    static ref P_DUBBY_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dubby-org",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // example.com.md: example.com, example.org
     static ref P_EXAMPLE_COM: Provider = Provider {
@@ -73,6 +163,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.example.com", port: 1337, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // fastmail.md: fastmail.com
@@ -83,6 +176,27 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/fastmail",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // five.chat.md: five.chat
+    static ref P_FIVE_CHAT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/five-chat",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // freenet.de.md: freenet.de
@@ -95,6 +209,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mx.freenet.de", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -107,6 +224,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.gmail.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -120,10 +240,36 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "mail.gmx.net", port: 465, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // hey.com.md: hey.com
+    static ref P_HEY_COM: Provider = Provider {
+        status: Status::BROKEN,
+        before_login_hint: "hey.com does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to hey.com.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/hey-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // i.ua.md: i.ua
-    // - skipping provider with status OK and no special things to do
+    static ref P_I_UA: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/i-ua",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // icloud.md: icloud.com, me.com, mac.com
     static ref P_ICLOUD: Provider = Provider {
@@ -135,19 +281,62 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.me.com", port: 993, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // kolst.com.md: kolst.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KOLST_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kolst-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // kontent.com.md: kontent.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KONTENT_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kontent-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAIL_RU: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mail-ru",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAILBOX_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mailbox-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // nauta.cu.md: nauta.cu
     static ref P_NAUTA_CU: Provider = Provider {
@@ -159,6 +348,17 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.nauta.cu", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.nauta.cu", port: 25, username_pattern: EMAIL },
         ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::DeleteServerAfter, value: "1" },
+            ConfigDefault { key: Config::BccSelf, value: "0" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+            ConfigDefault { key: Config::E2eeEnabled, value: "0" },
+            ConfigDefault { key: Config::MediaQuality, value: "1" },
+        ]),
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -171,9 +371,12 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap-mail.outlook.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // posteo.md: posteo.de
+    // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
     static ref P_POSTEO: Provider = Provider {
         status: Status::OK,
         before_login_hint: "",
@@ -183,6 +386,9 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "posteo.de", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -193,13 +399,49 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/protonmail",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // riseup.net.md: riseup.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_RISEUP_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/riseup-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // rogers.com.md: rogers.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_ROGERS_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/rogers-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // systemli.org.md: systemli.org
+    static ref P_SYSTEMLI_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/systemli-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
@@ -209,6 +451,9 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/t-online",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // testrun.md: testrun.org
@@ -222,6 +467,14 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // tiscali.it.md: tiscali.it
@@ -234,13 +487,36 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.tiscali.it", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // ukr.net.md: ukr.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_UKR_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/ukr-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // vfemail.md: vfemail.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_VFEMAIL: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/vfemail",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
     static ref P_WEB_DE: Provider = Provider {
@@ -253,21 +529,27 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.web.de", port: 143, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
     static ref P_YAHOO: Provider = Provider {
         status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with your Yahoo email address you have to allow \"less secure apps\" in the Yahoo webinterface.",
+        before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yahoo",
         server: vec![
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // yandex.ru.md: yandex.ru, yandex.com
+    // yandex.ru.md: yandex.com, yandex.by, yandex.kz, yandex.ru, yandex.ua, ya.ru, narod.ru
     static ref P_YANDEX_RU: Provider = Provider {
         status: Status::PREPARATION,
         before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
@@ -275,6 +557,9 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/yandex-ru",
         server: vec![
         ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -287,16 +572,27 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.ziggo.nl", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
+        ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("chello.at", &*P_CHELLO_AT),
+        ("xfinity.com", &*P_COMCAST),
+        ("comcast.net", &*P_COMCAST),
+        ("dismail.de", &*P_DISMAIL_DE),
+        ("disroot.org", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -309,9 +605,19 @@ lazy_static::lazy_static! {
         ("gmx.info", &*P_GMX_NET),
         ("gmx.biz", &*P_GMX_NET),
         ("gmx.com", &*P_GMX_NET),
+        ("hey.com", &*P_HEY_COM),
+        ("i.ua", &*P_I_UA),
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
+        ("kolst.com", &*P_KOLST_COM),
+        ("kontent.com", &*P_KONTENT_COM),
+        ("mail.ru", &*P_MAIL_RU),
+        ("inbox.ru", &*P_MAIL_RU),
+        ("bk.ru", &*P_MAIL_RU),
+        ("list.ru", &*P_MAIL_RU),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
@@ -319,12 +625,65 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
+        ("posteo.af", &*P_POSTEO),
+        ("posteo.at", &*P_POSTEO),
+        ("posteo.be", &*P_POSTEO),
+        ("posteo.ch", &*P_POSTEO),
+        ("posteo.cl", &*P_POSTEO),
+        ("posteo.co", &*P_POSTEO),
+        ("posteo.co.uk", &*P_POSTEO),
+        ("posteo.com.br", &*P_POSTEO),
+        ("posteo.cr", &*P_POSTEO),
+        ("posteo.cz", &*P_POSTEO),
+        ("posteo.dk", &*P_POSTEO),
+        ("posteo.ee", &*P_POSTEO),
+        ("posteo.es", &*P_POSTEO),
+        ("posteo.eu", &*P_POSTEO),
+        ("posteo.fi", &*P_POSTEO),
+        ("posteo.gl", &*P_POSTEO),
+        ("posteo.gr", &*P_POSTEO),
+        ("posteo.hn", &*P_POSTEO),
+        ("posteo.hr", &*P_POSTEO),
+        ("posteo.hu", &*P_POSTEO),
+        ("posteo.ie", &*P_POSTEO),
+        ("posteo.in", &*P_POSTEO),
+        ("posteo.is", &*P_POSTEO),
+        ("posteo.jp", &*P_POSTEO),
+        ("posteo.la", &*P_POSTEO),
+        ("posteo.li", &*P_POSTEO),
+        ("posteo.lt", &*P_POSTEO),
+        ("posteo.lu", &*P_POSTEO),
+        ("posteo.me", &*P_POSTEO),
+        ("posteo.mx", &*P_POSTEO),
+        ("posteo.my", &*P_POSTEO),
+        ("posteo.net", &*P_POSTEO),
+        ("posteo.nl", &*P_POSTEO),
+        ("posteo.no", &*P_POSTEO),
+        ("posteo.nz", &*P_POSTEO),
+        ("posteo.org", &*P_POSTEO),
+        ("posteo.pe", &*P_POSTEO),
+        ("posteo.pl", &*P_POSTEO),
+        ("posteo.pm", &*P_POSTEO),
+        ("posteo.pt", &*P_POSTEO),
+        ("posteo.ro", &*P_POSTEO),
+        ("posteo.ru", &*P_POSTEO),
+        ("posteo.se", &*P_POSTEO),
+        ("posteo.sg", &*P_POSTEO),
+        ("posteo.si", &*P_POSTEO),
+        ("posteo.tn", &*P_POSTEO),
+        ("posteo.uk", &*P_POSTEO),
+        ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("rogers.com", &*P_ROGERS_COM),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
+        ("ukr.net", &*P_UKR_NET),
+        ("vfemail.net", &*P_VFEMAIL),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),
         ("flirt.ms", &*P_WEB_DE),
@@ -367,8 +726,13 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
-        ("yandex.ru", &*P_YANDEX_RU),
         ("yandex.com", &*P_YANDEX_RU),
+        ("yandex.by", &*P_YANDEX_RU),
+        ("yandex.kz", &*P_YANDEX_RU),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("yandex.ua", &*P_YANDEX_RU),
+        ("ya.ru", &*P_YANDEX_RU),
+        ("narod.ru", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/provider/mod.rs

@@ -2,6 +2,7 @@
 
 mod data;
 
+use crate::config::Config;
 use crate::dc_tools::EmailAddress;
 use crate::provider::data::PROVIDER_DATA;
 
@@ -34,6 +35,13 @@ pub enum UsernamePattern {
     EMAILLOCALPART = 2,
 }
 
+#[derive(Debug, PartialEq)]
+#[repr(u8)]
+pub enum Oauth2Authorizer {
+    Yandex = 1,
+    Gmail = 2,
+}
+
 #[derive(Debug)]
 pub struct Server {
     pub protocol: Protocol,
@@ -57,6 +65,12 @@ impl Server {
     }
 }
 
+#[derive(Debug)]
+pub struct ConfigDefault {
+    pub key: Config,
+    pub value: &'static str,
+}
+
 #[derive(Debug)]
 pub struct Provider {
     pub status: Status,
@@ -64,6 +78,9 @@ pub struct Provider {
     pub after_login_hint: &'static str,
     pub overview_page: &'static str,
     pub server: Vec<Server>,
+    pub config_defaults: Option<Vec<ConfigDefault>>,
+    pub strict_tls: bool,
+    pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
 impl Provider {

src/provider/update.py

@@ -9,6 +9,9 @@ out_all = ""
 out_domains = ""
 domains_dict = {}
 
+def camel(name):
+    words = name.split("_")
+    return "".join(w.capitalize() for i, w in enumerate(words))
 
 def cleanstr(s):
     s = s.strip()
@@ -31,6 +34,18 @@ def file2url(f):
     return "https://providers.delta.chat/" + f
 
 
+def process_config_defaults(data):
+    if not "config_defaults" in data:
+        return "None"
+    defaults = "Some(vec![\n"
+    config_defaults = data.get("config_defaults", "")
+    for key in config_defaults:
+        value = str(config_defaults[key])
+        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
+    defaults += "        ])"
+    return defaults
+
+
 def process_data(data, file):
     status = data.get("status", "")
     if status != "OK" and status != "PREPARATION" and status != "BROKEN":
@@ -83,6 +98,14 @@ def process_data(data, file):
             server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
+    config_defaults = process_config_defaults(data)
+
+    strict_tls = data.get("strict_tls", False)
+    strict_tls = "true" if strict_tls else "false"
+
+    oauth2 = data.get("oauth2", "")
+    oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -93,6 +116,9 @@ def process_data(data, file):
         provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
+        provider += "        config_defaults: " + config_defaults + ",\n"
+        provider += "        strict_tls: " + strict_tls + ",\n"
+        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -103,11 +129,11 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "":
-        out_all += "    // - skipping provider with status OK and no special things to do\n\n"
-    else:
-        out_all += provider
-        out_domains += domains
+
+    # also add provider with no special things to do -
+    # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
+    out_all += provider
+    out_domains += domains
 
 
 def process_file(file):

src/qr.rs

@@ -2,7 +2,6 @@
 
 use lazy_static::lazy_static;
 use percent_encoding::percent_decode_str;
-use reqwest::Url;
 use serde::Deserialize;
 
 use crate::chat;
@@ -11,8 +10,7 @@ use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::key::dc_format_fingerprint;
-use crate::key::dc_normalize_fingerprint;
+use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
@@ -44,23 +42,23 @@ fn starts_with_ignore_case(string: &str, pattern: &str) -> bool {
 /// Check a scanned QR code.
 /// The function should be called after a QR code is scanned.
 /// The function takes the raw text scanned and checks what can be done with it.
-pub fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
+pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
     let qr = qr.as_ref();
 
     info!(context, "Scanned QR code: {}", qr);
 
     if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) {
-        decode_openpgp(context, qr)
+        decode_openpgp(context, qr).await
     } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
-        decode_mailto(context, qr)
+        decode_mailto(context, qr).await
     } else if qr.starts_with(SMTP_SCHEME) {
-        decode_smtp(context, qr)
+        decode_smtp(context, qr).await
     } else if qr.starts_with(MATMSG_SCHEME) {
-        decode_matmsg(context, qr)
+        decode_matmsg(context, qr).await
     } else if qr.starts_with(VCARD_SCHEME) {
-        decode_vcard(context, qr)
+        decode_vcard(context, qr).await
     } else if qr.starts_with(HTTP_SCHEME) || qr.starts_with(HTTPS_SCHEME) {
         Lot::from_url(qr)
     } else {
@@ -70,7 +68,7 @@ pub fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
 /// scheme: `OPENPGP4FPR:FINGERPRINT#a=ADDR&n=NAME&i=INVITENUMBER&s=AUTH`
 ///     or: `OPENPGP4FPR:FINGERPRINT#a=ADDR&g=GROUPNAME&x=GROUPID&i=INVITENUMBER&s=AUTH`
-fn decode_openpgp(context: &Context, qr: &str) -> Lot {
+async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
     let (fingerprint, fragment) = match payload.find('#').map(|offset| {
@@ -81,6 +79,14 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         Some(pair) => pair,
         None => (payload, ""),
     };
+    let fingerprint: Fingerprint = match fingerprint.parse() {
+        Ok(fp) => fp,
+        Err(err) => {
+            return Error::new(err)
+                .context("Failed to parse fingerprint in QR code")
+                .into()
+        }
+    };
 
     // replace & with \n to match expected param format
     let fragment = fragment.replace('&', "\n");
@@ -129,25 +135,13 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         None
     };
 
-    let fingerprint = dc_normalize_fingerprint(fingerprint);
-
-    // ensure valid fingerprint
-    if fingerprint.len() != 40 {
-        return format_err!("Bad fingerprint length in QR code").into();
-    }
-
-    println!(
-        "{:?} {:?} {:?} {:?} {:?} {:?} {:?}",
-        addr, name, invitenumber, auth, grpid, grpname, fingerprint
-    );
-
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
-    let peerstate = Peerstate::from_fingerprint(context, &context.sql, &fingerprint);
+    let peerstate = Peerstate::from_fingerprint(context, &fingerprint).await;
 
     if invitenumber.is_none() || auth.is_none() {
-        if let Some(peerstate) = peerstate {
+        if let Ok(peerstate) = peerstate {
             lot.state = LotState::QrFprOk;
 
             lot.id = Contact::add_or_lookup(
@@ -156,16 +150,18 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
                 peerstate.addr.clone(),
                 Origin::UnhandledQrScan,
             )
+            .await
             .map(|(id, _)| id)
             .unwrap_or_default();
 
             let (id, _) = chat::create_or_lookup_by_contact_id(context, lot.id, Blocked::Deaddrop)
+                .await
                 .unwrap_or_default();
 
-            chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr));
+            chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr)).await;
         } else {
             lot.state = LotState::QrFprWithoutAddr;
-            lot.text1 = Some(dc_format_fingerprint(&fingerprint));
+            lot.text1 = Some(fingerprint.to_string());
         }
     } else if let Some(addr) = addr {
         if grpid.is_some() && grpname.is_some() {
@@ -176,6 +172,7 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             lot.state = LotState::QrAskVerifyContact;
         }
         lot.id = Contact::add_or_lookup(context, &name, &addr, Origin::UnhandledQrScan)
+            .await
             .map(|(id, _)| id)
             .unwrap_or_default();
 
@@ -195,7 +192,7 @@ fn decode_account(_context: &Context, qr: &str) -> Lot {
 
     let mut lot = Lot::new();
 
-    if let Ok(url) = Url::parse(payload) {
+    if let Ok(url) = url::Url::parse(payload) {
         if url.scheme() == "https" {
             lot.state = LotState::QrAccount;
             lot.text1 = url.host_str().map(|x| x.to_string());
@@ -220,31 +217,22 @@ struct CreateAccountResponse {
 /// take a qr of the type DC_QR_ACCOUNT, parse it's parameters,
 /// download additional information from the contained url and set the parameters.
 /// on success, a configure::configure() should be able to log in to the account
-pub fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
 
-    let response = reqwest::blocking::Client::new().post(url_str).send();
+    let response: Result<CreateAccountResponse, surf::Error> =
+        surf::post(url_str).recv_json().await;
     if response.is_err() {
         bail!("Cannot create account, request to {} failed", url_str);
     }
-    let response = response.unwrap();
-    if !response.status().is_success() {
-        bail!("Request to {} unsuccessful: {:?}", url_str, response);
-    }
+    let parsed = response.unwrap();
 
-    let parsed: reqwest::Result<CreateAccountResponse> = response.json();
-    if parsed.is_err() {
-        bail!(
-            "Failed to parse JSON response from {}: error: {:?}",
-            url_str,
-            parsed
-        );
-    }
-    println!("response: {:?}", &parsed);
-    let parsed = parsed.unwrap();
-
-    context.set_config(Config::Addr, Some(&parsed.email))?;
-    context.set_config(Config::MailPw, Some(&parsed.password))?;
+    context
+        .set_config(Config::Addr, Some(&parsed.email))
+        .await?;
+    context
+        .set_config(Config::MailPw, Some(&parsed.password))
+        .await?;
 
     Ok(())
 }
@@ -252,7 +240,7 @@ pub fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
 /// Extract address for the mailto scheme.
 ///
 /// Scheme: `mailto:addr...?subject=...&body=..`
-fn decode_mailto(context: &Context, qr: &str) -> Lot {
+async fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let payload = &qr[MAILTO_SCHEME.len()..];
 
     let addr = if let Some(query_index) = payload.find('?') {
@@ -267,13 +255,13 @@ fn decode_mailto(context: &Context, qr: &str) -> Lot {
     };
 
     let name = "".to_string();
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 /// Extract address for the smtp scheme.
 ///
 /// Scheme: `SMTP:addr...:subject...:body...`
-fn decode_smtp(context: &Context, qr: &str) -> Lot {
+async fn decode_smtp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[SMTP_SCHEME.len()..];
 
     let addr = if let Some(query_index) = payload.find(':') {
@@ -287,7 +275,7 @@ fn decode_smtp(context: &Context, qr: &str) -> Lot {
         Err(err) => return err.into(),
     };
     let name = "".to_string();
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 /// Extract address for the matmsg scheme.
@@ -295,7 +283,7 @@ fn decode_smtp(context: &Context, qr: &str) -> Lot {
 /// Scheme: `MATMSG:TO:addr...;SUB:subject...;BODY:body...;`
 ///
 /// There may or may not be linebreaks after the fields.
-fn decode_matmsg(context: &Context, qr: &str) -> Lot {
+async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     // Does not work when the text `TO:` is used in subject/body _and_ TO: is not the first field.
     // we ignore this case.
     let addr = if let Some(to_index) = qr.find("TO:") {
@@ -315,7 +303,7 @@ fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     };
 
     let name = "".to_string();
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 lazy_static! {
@@ -328,7 +316,7 @@ lazy_static! {
 /// Extract address for the matmsg scheme.
 ///
 /// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
-fn decode_vcard(context: &Context, qr: &str) -> Lot {
+async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
         .captures(qr)
         .map(|caps| {
@@ -348,7 +336,7 @@ fn decode_vcard(context: &Context, qr: &str) -> Lot {
         return format_err!("Bad e-mail address").into();
     };
 
-    Lot::from_address(context, name, addr)
+    Lot::from_address(context, name, addr).await
 }
 
 impl Lot {
@@ -368,10 +356,10 @@ impl Lot {
         l
     }
 
-    pub fn from_address(context: &Context, name: String, addr: String) -> Self {
+    pub async fn from_address(context: &Context, name: String, addr: String) -> Self {
         let mut l = Lot::new();
         l.state = LotState::QrAddr;
-        l.id = match Contact::add_or_lookup(context, name, addr, Origin::UnhandledQrScan) {
+        l.id = match Contact::add_or_lookup(context, name, addr, Origin::UnhandledQrScan).await {
             Ok((id, _)) => id,
             Err(err) => return err.into(),
         };
@@ -395,13 +383,13 @@ fn normalize_address(addr: &str) -> Result<String, Error> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
-    #[test]
-    fn test_decode_http() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_http() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "http://www.hello.com");
+        let res = check_qr(&ctx.ctx, "http://www.hello.com").await;
 
         assert_eq!(res.get_state(), LotState::QrUrl);
         assert_eq!(res.get_id(), 0);
@@ -409,11 +397,11 @@ mod tests {
         assert!(res.get_text2().is_none());
     }
 
-    #[test]
-    fn test_decode_https() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_https() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "https://www.hello.com");
+        let res = check_qr(&ctx.ctx, "https://www.hello.com").await;
 
         assert_eq!(res.get_state(), LotState::QrUrl);
         assert_eq!(res.get_id(), 0);
@@ -421,11 +409,11 @@ mod tests {
         assert!(res.get_text2().is_none());
     }
 
-    #[test]
-    fn test_decode_text() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_text() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "I am so cool");
+        let res = check_qr(&ctx.ctx, "I am so cool").await;
 
         assert_eq!(res.get_state(), LotState::QrText);
         assert_eq!(res.get_id(), 0);
@@ -433,88 +421,90 @@ mod tests {
         assert!(res.get_text2().is_none());
     }
 
-    #[test]
-    fn test_decode_vcard() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_vcard() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "BEGIN:VCARD\nVERSION:3.0\nN:Last;First\nEMAIL;TYPE=INTERNET:stress@test.local\nEND:VCARD"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
         assert_eq!(contact.get_name(), "First Last");
     }
 
-    #[test]
-    fn test_decode_matmsg() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_matmsg() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "MATMSG:TO:\n\nstress@test.local ; \n\nSUB:\n\nSubject here\n\nBODY:\n\nhelloworld\n;;",
-        );
+        )
+        .await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
     }
 
-    #[test]
-    fn test_decode_mailto() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_mailto() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "mailto:stress@test.local?subject=hello&body=world",
-        );
+        )
+        .await;
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
 
-        let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org");
+        let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org").await;
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "no-questionmark@example.org");
 
-        let res = check_qr(&ctx.ctx, "mailto:no-addr");
+        let res = check_qr(&ctx.ctx, "mailto:no-addr").await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
     }
 
-    #[test]
-    fn test_decode_smtp() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_smtp() {
+        let ctx = TestContext::new().await;
 
-        let res = check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld");
+        let res = check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld").await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
     }
 
-    #[test]
-    fn test_decode_openpgp_group() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_openpgp_group() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
@@ -525,25 +515,25 @@ mod tests {
         let res = check_qr(
             &ctx.ctx,
             "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
         assert_ne!(res.get_id(), 0);
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
     }
 
-    #[test]
-    fn test_decode_openpgp_secure_join() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_openpgp_secure_join() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
@@ -553,25 +543,26 @@ mod tests {
         let res = check_qr(
             &ctx.ctx,
             "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
-        );
+        ).await;
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
         assert_ne!(res.get_id(), 0);
 
-        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
     }
 
-    #[test]
-    fn test_decode_openpgp_without_addr() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_openpgp_without_addr() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "OPENPGP4FPR:1234567890123456789012345678901234567890",
-        );
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
         assert_eq!(
             res.get_text1().unwrap(),
@@ -584,7 +575,8 @@ mod tests {
         let res = check_qr(
             &ctx.ctx,
             "openpgp4fpr:1234567890123456789012345678901234567890",
-        );
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
         assert_eq!(
             res.get_text1().unwrap(),
@@ -592,19 +584,20 @@ mod tests {
         );
         assert_eq!(res.get_id(), 0);
 
-        let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890");
+        let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890").await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert_eq!(res.get_id(), 0);
     }
 
-    #[test]
-    fn test_decode_account() {
-        let ctx = dummy_context();
+    #[async_std::test]
+    async fn test_decode_account() {
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
             "DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrAccount);
         assert_eq!(res.get_text1().unwrap(), "example.org");
 
@@ -612,19 +605,20 @@ mod tests {
         let res = check_qr(
             &ctx.ctx,
             "dcaccount:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrAccount);
         assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
-    #[test]
-    fn test_decode_account_bad_scheme() {
-        let ctx = dummy_context();
-
+    #[async_std::test]
+    async fn test_decode_account_bad_scheme() {
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
             "DCACCOUNT:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
 
@@ -632,7 +626,8 @@ mod tests {
         let res = check_qr(
             &ctx.ctx,
             "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
+        )
+        .await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
     }

src/scheduler.rs

@@ -0,0 +1,550 @@
+#![warn(clippy::indexing_slicing)]
+
+use async_std::prelude::*;
+use async_std::sync::{channel, Receiver, Sender};
+use async_std::task;
+
+use crate::context::Context;
+use crate::imap::Imap;
+use crate::job::{self, Thread};
+use crate::{config::Config, message::MsgId, smtp::Smtp};
+
+pub(crate) struct StopToken;
+
+/// Job and connection scheduler.
+#[derive(Debug)]
+#[allow(clippy::large_enum_variant)]
+pub(crate) enum Scheduler {
+    Stopped,
+    Running {
+        inbox: ImapConnectionState,
+        inbox_handle: Option<task::JoinHandle<()>>,
+        mvbox: ImapConnectionState,
+        mvbox_handle: Option<task::JoinHandle<()>>,
+        sentbox: ImapConnectionState,
+        sentbox_handle: Option<task::JoinHandle<()>>,
+        smtp: SmtpConnectionState,
+        smtp_handle: Option<task::JoinHandle<()>>,
+    },
+}
+
+impl Context {
+    /// Indicate that the network likely has come back.
+    pub async fn maybe_network(&self) {
+        self.scheduler.read().await.maybe_network().await;
+    }
+
+    pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_inbox(info).await;
+    }
+
+    pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_smtp(info).await;
+    }
+}
+
+async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers) {
+    use futures::future::FutureExt;
+
+    info!(ctx, "starting inbox loop");
+    let ImapConnectionHandlers {
+        mut connection,
+        stop_receiver,
+        shutdown_sender,
+    } = inbox_handlers;
+
+    let ctx1 = ctx.clone();
+    let fut = async move {
+        started.send(()).await;
+        let ctx = ctx1;
+
+        // track number of continously executed jobs
+        let mut jobs_loaded = 0;
+        let mut info = InterruptInfo::default();
+        loop {
+            match job::load_next(&ctx, Thread::Imap, &info).await {
+                Some(job) if jobs_loaded <= 20 => {
+                    jobs_loaded += 1;
+                    job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
+                    info = Default::default();
+                }
+                Some(job) => {
+                    // Let the fetch run, but return back to the job afterwards.
+                    info!(ctx, "postponing imap-job {} to run fetch...", job);
+                    jobs_loaded = 0;
+                    fetch(&ctx, &mut connection).await;
+                }
+                None => {
+                    jobs_loaded = 0;
+                    info = if ctx.get_config_bool(Config::InboxWatch).await {
+                        fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
+                    } else {
+                        connection.fake_idle(&ctx, None).await
+                    };
+                }
+            }
+        }
+    };
+
+    stop_receiver
+        .recv()
+        .map(|_| {
+            info!(ctx, "shutting down inbox loop");
+        })
+        .race(fut)
+        .await;
+    shutdown_sender.send(()).await;
+}
+
+async fn fetch(ctx: &Context, connection: &mut Imap) {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
+        Some(watch_folder) => {
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "{}", err);
+                return;
+            }
+
+            // fetch
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
+        }
+        None => {
+            warn!(ctx, "Can not fetch inbox folder, not set");
+            connection.fake_idle(&ctx, None).await;
+        }
+    }
+}
+
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
+    match ctx.get_config(folder).await {
+        Some(watch_folder) => {
+            // connect and fake idle if unable to connect
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "imap connection failed: {}", err);
+                return connection.fake_idle(&ctx, None).await;
+            }
+
+            // fetch
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
+
+            // idle
+            if connection.can_idle() {
+                connection
+                    .idle(&ctx, Some(watch_folder))
+                    .await
+                    .unwrap_or_else(|err| {
+                        connection.trigger_reconnect();
+                        warn!(ctx, "{}", err);
+                        InterruptInfo::new(false, None)
+                    })
+            } else {
+                connection.fake_idle(&ctx, Some(watch_folder)).await
+            }
+        }
+        None => {
+            warn!(ctx, "Can not watch {} folder, not set", folder);
+            connection.fake_idle(&ctx, None).await
+        }
+    }
+}
+
+async fn simple_imap_loop(
+    ctx: Context,
+    started: Sender<()>,
+    inbox_handlers: ImapConnectionHandlers,
+    folder: Config,
+) {
+    use futures::future::FutureExt;
+
+    info!(ctx, "starting simple loop for {}", folder.as_ref());
+    let ImapConnectionHandlers {
+        mut connection,
+        stop_receiver,
+        shutdown_sender,
+    } = inbox_handlers;
+
+    let ctx1 = ctx.clone();
+
+    let fut = async move {
+        started.send(()).await;
+        let ctx = ctx1;
+
+        loop {
+            fetch_idle(&ctx, &mut connection, folder).await;
+        }
+    };
+
+    stop_receiver
+        .recv()
+        .map(|_| {
+            info!(ctx, "shutting down simple loop");
+        })
+        .race(fut)
+        .await;
+    shutdown_sender.send(()).await;
+}
+
+async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
+    use futures::future::FutureExt;
+
+    info!(ctx, "starting smtp loop");
+    let SmtpConnectionHandlers {
+        mut connection,
+        stop_receiver,
+        shutdown_sender,
+        idle_interrupt_receiver,
+    } = smtp_handlers;
+
+    let ctx1 = ctx.clone();
+    let fut = async move {
+        started.send(()).await;
+        let ctx = ctx1;
+
+        let mut interrupt_info = Default::default();
+        loop {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
+                Some(job) => {
+                    info!(ctx, "executing smtp job");
+                    job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
+                    interrupt_info = Default::default();
+                }
+                None => {
+                    // Fake Idle
+                    info!(ctx, "smtp fake idle - started");
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    info!(ctx, "smtp fake idle - interrupted")
+                }
+            }
+        }
+    };
+
+    stop_receiver
+        .recv()
+        .map(|_| {
+            info!(ctx, "shutting down smtp loop");
+        })
+        .race(fut)
+        .await;
+    shutdown_sender.send(()).await;
+}
+
+impl Scheduler {
+    /// Start the scheduler, panics if it is already running.
+    pub async fn start(&mut self, ctx: Context) {
+        let (mvbox, mvbox_handlers) = ImapConnectionState::new();
+        let (sentbox, sentbox_handlers) = ImapConnectionState::new();
+        let (smtp, smtp_handlers) = SmtpConnectionState::new();
+        let (inbox, inbox_handlers) = ImapConnectionState::new();
+
+        let (inbox_start_send, inbox_start_recv) = channel(1);
+        let (mvbox_start_send, mvbox_start_recv) = channel(1);
+        let mut mvbox_handle = None;
+        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let mut sentbox_handle = None;
+        let (smtp_start_send, smtp_start_recv) = channel(1);
+
+        let ctx1 = ctx.clone();
+        let inbox_handle = Some(task::spawn(async move {
+            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
+        }));
+
+        if ctx.get_config_bool(Config::MvboxWatch).await {
+            let ctx1 = ctx.clone();
+            mvbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    mvbox_start_send,
+                    mvbox_handlers,
+                    Config::ConfiguredMvboxFolder,
+                )
+                .await
+            }));
+        } else {
+            mvbox_start_send.send(()).await;
+        }
+
+        if ctx.get_config_bool(Config::SentboxWatch).await {
+            let ctx1 = ctx.clone();
+            sentbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    sentbox_start_send,
+                    sentbox_handlers,
+                    Config::ConfiguredSentboxFolder,
+                )
+                .await
+            }));
+        } else {
+            sentbox_start_send.send(()).await;
+        }
+
+        let ctx1 = ctx.clone();
+        let smtp_handle = Some(task::spawn(async move {
+            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
+        }));
+
+        *self = Scheduler::Running {
+            inbox,
+            mvbox,
+            sentbox,
+            smtp,
+            inbox_handle,
+            mvbox_handle,
+            sentbox_handle,
+            smtp_handle,
+        };
+
+        // wait for all loops to be started
+        if let Err(err) = inbox_start_recv
+            .recv()
+            .try_join(mvbox_start_recv.recv())
+            .try_join(sentbox_start_recv.recv())
+            .try_join(smtp_start_recv.recv())
+            .await
+        {
+            error!(ctx, "failed to start scheduler: {}", err);
+        }
+
+        info!(ctx, "scheduler is running");
+    }
+
+    async fn maybe_network(&self) {
+        if !self.is_running() {
+            return;
+        }
+
+        self.interrupt_inbox(InterruptInfo::new(true, None))
+            .join(self.interrupt_mvbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_smtp(InterruptInfo::new(true, None)))
+            .await;
+    }
+
+    async fn interrupt_inbox(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref inbox, .. } = self {
+            inbox.interrupt(info).await;
+        }
+    }
+
+    async fn interrupt_mvbox(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref mvbox, .. } = self {
+            mvbox.interrupt(info).await;
+        }
+    }
+
+    async fn interrupt_sentbox(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref sentbox, .. } = self {
+            sentbox.interrupt(info).await;
+        }
+    }
+
+    async fn interrupt_smtp(&self, info: InterruptInfo) {
+        if let Scheduler::Running { ref smtp, .. } = self {
+            smtp.interrupt(info).await;
+        }
+    }
+
+    /// Halts the scheduler, must be called first, and then `stop`.
+    pub(crate) async fn pre_stop(&self) -> StopToken {
+        match self {
+            Scheduler::Stopped => {
+                panic!("WARN: already stopped");
+            }
+            Scheduler::Running {
+                inbox,
+                mvbox,
+                sentbox,
+                smtp,
+                ..
+            } => {
+                inbox
+                    .stop()
+                    .join(mvbox.stop())
+                    .join(sentbox.stop())
+                    .join(smtp.stop())
+                    .await;
+
+                StopToken
+            }
+        }
+    }
+
+    /// Halt the scheduler, must only be called after pre_stop.
+    pub(crate) async fn stop(&mut self, _t: StopToken) {
+        match self {
+            Scheduler::Stopped => {
+                panic!("WARN: already stopped");
+            }
+            Scheduler::Running {
+                inbox_handle,
+                mvbox_handle,
+                sentbox_handle,
+                smtp_handle,
+                ..
+            } => {
+                if let Some(handle) = inbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = mvbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = sentbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = smtp_handle.take() {
+                    handle.await;
+                }
+
+                *self = Scheduler::Stopped;
+            }
+        }
+    }
+
+    /// Check if the scheduler is running.
+    pub fn is_running(&self) -> bool {
+        match self {
+            Scheduler::Running { .. } => true,
+            _ => false,
+        }
+    }
+}
+
+/// Connection state logic shared between imap and smtp connections.
+#[derive(Debug)]
+struct ConnectionState {
+    /// Channel to notify that shutdown has completed.
+    shutdown_receiver: Receiver<()>,
+    /// Channel to interrupt the whole connection.
+    stop_sender: Sender<()>,
+    /// Channel to interrupt idle.
+    idle_interrupt_sender: Sender<InterruptInfo>,
+}
+
+impl ConnectionState {
+    /// Shutdown this connection completely.
+    async fn stop(&self) {
+        // Trigger shutdown of the run loop.
+        self.stop_sender.send(()).await;
+        // Wait for a notification that the run loop has been shutdown.
+        self.shutdown_receiver.recv().await.ok();
+    }
+
+    async fn interrupt(&self, info: InterruptInfo) {
+        // Use try_send to avoid blocking on interrupts.
+        self.idle_interrupt_sender.try_send(info).ok();
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct SmtpConnectionState {
+    state: ConnectionState,
+}
+
+impl SmtpConnectionState {
+    fn new() -> (Self, SmtpConnectionHandlers) {
+        let (stop_sender, stop_receiver) = channel(1);
+        let (shutdown_sender, shutdown_receiver) = channel(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+
+        let handlers = SmtpConnectionHandlers {
+            connection: Smtp::new(),
+            stop_receiver,
+            shutdown_sender,
+            idle_interrupt_receiver,
+        };
+
+        let state = ConnectionState {
+            idle_interrupt_sender,
+            shutdown_receiver,
+            stop_sender,
+        };
+
+        let conn = SmtpConnectionState { state };
+
+        (conn, handlers)
+    }
+
+    /// Interrupt any form of idle.
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
+    }
+
+    /// Shutdown this connection completely.
+    async fn stop(&self) {
+        self.state.stop().await;
+    }
+}
+
+struct SmtpConnectionHandlers {
+    connection: Smtp,
+    stop_receiver: Receiver<()>,
+    shutdown_sender: Sender<()>,
+    idle_interrupt_receiver: Receiver<InterruptInfo>,
+}
+
+#[derive(Debug)]
+pub(crate) struct ImapConnectionState {
+    state: ConnectionState,
+}
+
+impl ImapConnectionState {
+    /// Construct a new connection.
+    fn new() -> (Self, ImapConnectionHandlers) {
+        let (stop_sender, stop_receiver) = channel(1);
+        let (shutdown_sender, shutdown_receiver) = channel(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+
+        let handlers = ImapConnectionHandlers {
+            connection: Imap::new(idle_interrupt_receiver),
+            stop_receiver,
+            shutdown_sender,
+        };
+
+        let state = ConnectionState {
+            idle_interrupt_sender,
+            shutdown_receiver,
+            stop_sender,
+        };
+
+        let conn = ImapConnectionState { state };
+
+        (conn, handlers)
+    }
+
+    /// Interrupt any form of idle.
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
+    }
+
+    /// Shutdown this connection completely.
+    async fn stop(&self) {
+        self.state.stop().await;
+    }
+}
+
+#[derive(Debug)]
+struct ImapConnectionHandlers {
+    connection: Imap,
+    stop_receiver: Receiver<()>,
+    shutdown_sender: Sender<()>,
+}
+
+#[derive(Default, Debug)]
+pub struct InterruptInfo {
+    pub probe_network: bool,
+    pub msg_id: Option<MsgId>,
+}
+
+impl InterruptInfo {
+    pub fn new(probe_network: bool, msg_id: Option<MsgId>) -> Self {
+        Self {
+            probe_network,
+            msg_id,
+        }
+    }
+}

src/securejoin.rs

@@ -1,5 +1,7 @@
 //! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol)
 
+use std::time::Duration;
+
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
@@ -12,7 +14,7 @@ use crate::e2ee::*;
 use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, DcKey, Key, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -30,7 +32,7 @@ macro_rules! joiner_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.call_cb($crate::events::Event::SecurejoinJoinerProgress {
+        $context.emit_event($crate::events::Event::SecurejoinJoinerProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -43,7 +45,7 @@ macro_rules! inviter_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.call_cb($crate::events::Event::SecurejoinInviterProgress {
+        $context.emit_event($crate::events::Event::SecurejoinInviterProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -55,7 +57,7 @@ macro_rules! get_qr_attr {
         $context
             .bob
             .read()
-            .unwrap()
+            .await
             .qr_scan
             .as_ref()
             .unwrap()
@@ -65,21 +67,20 @@ macro_rules! get_qr_attr {
     };
 }
 
-pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<String> {
+pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<String> {
     /*=======================================================
     ====             Alice - the inviter side            ====
     ====   Step 1 in "Setup verified contact" protocol   ====
     =======================================================*/
 
-    let fingerprint: String;
-
-    ensure_secret_key_exists(context).ok();
+    ensure_secret_key_exists(context).await.ok();
 
     // invitenumber will be used to allow starting the handshake,
     // auth will be used to verify the fingerprint
-    let invitenumber = token::lookup_or_new(context, token::Namespace::InviteNumber, group_chat_id);
-    let auth = token::lookup_or_new(context, token::Namespace::Auth, group_chat_id);
-    let self_addr = match context.get_config(Config::ConfiguredAddr) {
+    let invitenumber =
+        token::lookup_or_new(context, token::Namespace::InviteNumber, group_chat_id).await;
+    let auth = token::lookup_or_new(context, token::Namespace::Auth, group_chat_id).await;
+    let self_addr = match context.get_config(Config::ConfiguredAddr).await {
         Some(addr) => addr,
         None => {
             error!(context, "Not configured, cannot generate QR code.",);
@@ -87,9 +88,12 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<
         }
     };
 
-    let self_name = context.get_config(Config::Displayname).unwrap_or_default();
+    let self_name = context
+        .get_config(Config::Displayname)
+        .await
+        .unwrap_or_default();
 
-    fingerprint = match get_self_fingerprint(context) {
+    let fingerprint: Fingerprint = match get_self_fingerprint(context).await {
         Some(fp) => fp,
         None => {
             return None;
@@ -103,14 +107,14 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<
 
     let qr = if !group_chat_id.is_unset() {
         // parameters used: a=g=x=i=s=
-        if let Ok(chat) = Chat::load_from_db(context, group_chat_id) {
+        if let Ok(chat) = Chat::load_from_db(context, group_chat_id).await {
             let group_name = chat.get_name();
             let group_name_urlencoded =
                 utf8_percent_encode(&group_name, NON_ALPHANUMERIC).to_string();
 
             Some(format!(
                 "OPENPGP4FPR:{}#a={}&g={}&x={}&i={}&s={}",
-                fingerprint,
+                fingerprint.hex(),
                 self_addr_urlencoded,
                 &group_name_urlencoded,
                 &chat.grpid,
@@ -125,7 +129,11 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<
         // parameters used: a=n=i=s=
         Some(format!(
             "OPENPGP4FPR:{}#a={}&n={}&i={}&s={}",
-            fingerprint, self_addr_urlencoded, self_name_urlencoded, &invitenumber, &auth,
+            fingerprint.hex(),
+            self_addr_urlencoded,
+            self_name_urlencoded,
+            &invitenumber,
+            &auth,
         ))
     };
 
@@ -134,9 +142,9 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<
     qr
 }
 
-fn get_self_fingerprint(context: &Context) -> Option<String> {
-    match SignedPublicKey::load_self(context) {
-        Ok(key) => Some(Key::from(key).fingerprint()),
+async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
+    match SignedPublicKey::load_self(context).await {
+        Ok(key) => Some(key.fingerprint()),
         Err(_) => {
             warn!(context, "get_self_fingerprint(): failed to load key");
             None
@@ -146,74 +154,72 @@ fn get_self_fingerprint(context: &Context) -> Option<String> {
 
 /// Take a scanned QR-code and do the setup-contact/join-group handshake.
 /// See the ffi-documentation for more details.
-pub fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
-    let cleanup =
-        |context: &Context, contact_chat_id: ChatId, ongoing_allocated: bool, join_vg: bool| {
-            let mut bob = context.bob.write().unwrap();
-            bob.expects = 0;
-            let ret_chat_id: ChatId = if bob.status == DC_BOB_SUCCESS {
-                if join_vg {
-                    chat::get_chat_id_by_grpid(
-                        context,
-                        bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap(),
-                    )
-                    .unwrap_or((ChatId::new(0), false, Blocked::Not))
-                    .0
-                } else {
-                    contact_chat_id
-                }
-            } else {
-                ChatId::new(0)
-            };
-            bob.qr_scan = None;
-
-            if ongoing_allocated {
-                context.free_ongoing();
-            }
-            ret_chat_id
-        };
-
-    /*========================================================
-    ====             Bob - the joiner's side             =====
-    ====   Step 2 in "Setup verified contact" protocol   =====
-    ========================================================*/
-
-    let mut contact_chat_id = ChatId::new(0);
-    let mut join_vg: bool = false;
-
-    info!(context, "Requesting secure-join ...",);
-    ensure_secret_key_exists(context).ok();
-    if !context.alloc_ongoing() {
-        return cleanup(&context, contact_chat_id, false, join_vg);
+pub async fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
+    if let Err(err) = context.alloc_ongoing().await {
+        error!(context, "SecureJoin Error: {}", err);
+        return ChatId::new(0);
     }
-    let qr_scan = check_qr(context, &qr);
+
+    match securejoin(context, qr).await {
+        Ok(id) => id,
+        Err(err) => {
+            error!(context, "SecureJoin Error: {}", err);
+            ChatId::new(0)
+        }
+    }
+}
+
+///             Bob - the joiner's side
+///   Step 2 in "Setup verified contact" protocol
+async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, anyhow::Error> {
+    struct DropGuard<'a> {
+        context: &'a Context,
+    }
+
+    impl Drop for DropGuard<'_> {
+        fn drop(&mut self) {
+            async_std::task::block_on(async {
+                let mut bob = self.context.bob.write().await;
+                bob.expects = 0;
+                bob.qr_scan = None;
+                self.context.free_ongoing().await;
+            });
+        }
+    }
+
+    let _guard = DropGuard { context: &context };
+
+    info!(context, "Requesting secure-join ...");
+    ensure_secret_key_exists(context).await.ok();
+    let qr_scan = check_qr(context, &qr).await;
     if qr_scan.state != LotState::QrAskVerifyContact && qr_scan.state != LotState::QrAskVerifyGroup
     {
-        error!(context, "Unknown QR code.",);
-        return cleanup(&context, contact_chat_id, true, join_vg);
+        bail!("Unknown QR code.");
     }
-    contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id) {
+    let contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id).await {
         Ok(chat_id) => chat_id,
         Err(_) => {
-            error!(context, "Unknown contact.");
-            return cleanup(&context, contact_chat_id, true, join_vg);
+            bail!("Unknown contact.");
         }
     };
-    if context.shall_stop_ongoing() {
-        return cleanup(&context, contact_chat_id, true, join_vg);
+
+    if context.shall_stop_ongoing().await {
+        bail!("Interrupted");
     }
-    join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
+
+    let join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
     {
-        let mut bob = context.bob.write().unwrap();
+        let mut bob = context.bob.write().await;
         bob.status = 0;
         bob.qr_scan = Some(qr_scan);
     }
-    if fingerprint_equals_sender(
+
+    let fp_equals_sender = fingerprint_equals_sender(
         context,
         context
             .bob
             .read()
-            .unwrap()
+            .await
             .qr_scan
             .as_ref()
             .unwrap()
@@ -221,13 +227,20 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
             .as_ref()
             .unwrap(),
         contact_chat_id,
-    ) {
+    )
+    .await?;
+
+    if fp_equals_sender {
         // the scanned fingerprint matches Alice's key,
         // we can proceed to step 4b) directly and save two mails
         info!(context, "Taking protocol shortcut.");
-        context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
-        joiner_progress!(context, chat_id_2_contact_id(context, contact_chat_id), 400);
-        let own_fingerprint = get_self_fingerprint(context).unwrap_or_default();
+        context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
+        joiner_progress!(
+            context,
+            chat_id_2_contact_id(context, contact_chat_id).await?,
+            400
+        );
+        let own_fingerprint = get_self_fingerprint(context).await;
 
         // Bob -> Alice
         send_handshake_msg(
@@ -239,15 +252,16 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
                 "vc-request-with-auth"
             },
             get_qr_attr!(context, auth).to_string(),
-            Some(own_fingerprint),
+            own_fingerprint,
             if join_vg {
                 get_qr_attr!(context, text2).to_string()
             } else {
                 "".to_string()
             },
-        );
+        )
+        .await?;
     } else {
-        context.bob.write().unwrap().expects = DC_VC_AUTH_REQUIRED;
+        context.bob.write().await.expects = DC_VC_AUTH_REQUIRED;
 
         // Bob -> Alice
         send_handshake_msg(
@@ -257,31 +271,43 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
             get_qr_attr!(context, invitenumber),
             None,
             "",
-        );
+        )
+        .await?;
     }
 
     if join_vg {
         // for a group-join, wait until the secure-join is done and the group is created
-        while !context.shall_stop_ongoing() {
-            std::thread::sleep(std::time::Duration::from_millis(200));
+        while !context.shall_stop_ongoing().await {
+            async_std::task::sleep(Duration::from_millis(50)).await;
+        }
+        let bob = context.bob.read().await;
+        if bob.status == DC_BOB_SUCCESS {
+            let id = chat::get_chat_id_by_grpid(
+                context,
+                bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap(),
+            )
+            .await?
+            .0;
+            Ok(id)
+        } else {
+            bail!("Failed to join");
         }
-        cleanup(&context, contact_chat_id, true, join_vg)
     } else {
         // for a one-to-one-chat, the chat is already known, return the chat-id,
         // the verification runs in background
-        context.free_ongoing();
-        contact_chat_id
+        context.free_ongoing().await;
+        Ok(contact_chat_id)
     }
 }
 
-fn send_handshake_msg(
+async fn send_handshake_msg(
     context: &Context,
     contact_chat_id: ChatId,
     step: &str,
     param2: impl AsRef<str>,
-    fingerprint: Option<String>,
+    fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
-) {
+) -> Result<(), HandshakeError> {
     let mut msg = Message::default();
     msg.viewtype = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
@@ -296,7 +322,7 @@ fn send_handshake_msg(
         msg.param.set(Param::Arg2, param2);
     }
     if let Some(fp) = fingerprint {
-        msg.param.set(Param::Arg3, fp);
+        msg.param.set(Param::Arg3, fp.hex());
     }
     if !grpid.as_ref().is_empty() {
         msg.param.set(Param::Arg4, grpid.as_ref());
@@ -309,41 +335,45 @@ fn send_handshake_msg(
     } else {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
     }
-    // TODO. handle cleanup on error
-    chat::send_msg(context, contact_chat_id, &mut msg).unwrap_or_default();
+
+    chat::send_msg(context, contact_chat_id, &mut msg)
+        .await
+        .map_err(HandshakeError::MsgSendFailed)?;
+
+    Ok(())
 }
 
-fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32 {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id);
+async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> Result<u32, Error> {
+    let contacts = chat::get_chat_contacts(context, contact_chat_id).await?;
     if contacts.len() == 1 {
-        contacts[0]
+        Ok(contacts[0])
     } else {
-        0
+        Ok(0)
     }
 }
 
-fn fingerprint_equals_sender(
+async fn fingerprint_equals_sender(
     context: &Context,
-    fingerprint: impl AsRef<str>,
+    fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
-) -> bool {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id);
+) -> Result<bool, Error> {
+    let contacts = chat::get_chat_contacts(context, contact_chat_id).await?;
 
     if contacts.len() == 1 {
-        if let Ok(contact) = Contact::load_from_db(context, contacts[0]) {
-            if let Some(peerstate) = Peerstate::from_addr(context, &context.sql, contact.get_addr())
-            {
-                let fingerprint_normalized = dc_normalize_fingerprint(fingerprint.as_ref());
+        if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
+            if let Ok(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
                 if peerstate.public_key_fingerprint.is_some()
-                    && &fingerprint_normalized == peerstate.public_key_fingerprint.as_ref().unwrap()
+                    && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
-                    return true;
+                    return Ok(true);
                 }
             }
         }
     }
-    false
+
+    Ok(false)
 }
+
 #[derive(Debug, thiserror::Error)]
 pub(crate) enum HandshakeError {
     #[error("Can not be called with special contact ID")]
@@ -360,6 +390,12 @@ pub(crate) enum HandshakeError {
     ChatNotFound { group: String },
     #[error("No configured self address found")]
     NoSelfAddr,
+    #[error("Failed to send message")]
+    MsgSendFailed(#[source] Error),
+    #[error("Failed to parse fingerprint")]
+    BadFingerprint(#[from] crate::key::FingerprintError),
+    #[error("{0}")]
+    Other(#[from] Error),
 }
 
 /// What to do with a Secure-Join handshake message after it was handled.
@@ -386,7 +422,7 @@ pub(crate) enum HandshakeMessage {
 /// When handle_securejoin_handshake() is called,
 /// the message is not yet filed in the database;
 /// this is done by receive_imf() later on as needed.
-pub(crate) fn handle_securejoin_handshake(
+pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
     mime_message: &MimeMessage,
     contact_id: u32,
@@ -404,10 +440,10 @@ pub(crate) fn handle_securejoin_handshake(
     );
 
     let contact_chat_id =
-        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not) {
+        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not).await {
             Ok((chat_id, blocked)) => {
                 if blocked != Blocked::Not {
-                    chat_id.unblock(context);
+                    chat_id.unblock(context).await;
                 }
                 chat_id
             }
@@ -439,7 +475,7 @@ pub(crate) fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !token::exists(context, token::Namespace::InviteNumber, &invitenumber) {
+            if !token::exists(context, token::Namespace::InviteNumber, &invitenumber).await {
                 warn!(context, "Secure-join denied (bad invitenumber).");
                 return Ok(HandshakeMessage::Ignore);
             }
@@ -455,7 +491,8 @@ pub(crate) fn handle_securejoin_handshake(
                 "",
                 None,
                 "",
-            );
+            )
+            .await?;
             Ok(HandshakeMessage::Done)
         }
         "vg-auth-required" | "vc-auth-required" => {
@@ -466,7 +503,7 @@ pub(crate) fn handle_securejoin_handshake(
 
             // verify that Alice's Autocrypt key and fingerprint matches the QR-code
             let cond = {
-                let bob = context.bob.read().unwrap();
+                let bob = context.bob.read().await;
                 let scan = bob.qr_scan.as_ref();
                 scan.is_none()
                     || bob.expects != DC_VC_AUTH_REQUIRED
@@ -478,10 +515,11 @@ pub(crate) fn handle_securejoin_handshake(
                 // no error, just aborted somehow or a mail from another handshake
                 return Ok(HandshakeMessage::Ignore);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
             let auth = get_qr_attr!(context, auth).to_string();
 
-            if !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice) {
+            if !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -490,25 +528,29 @@ pub(crate) fn handle_securejoin_handshake(
                     } else {
                         "Not encrypted."
                     },
-                );
-                context.bob.write().unwrap().status = 0; // secure-join failed
-                context.stop_ongoing();
+                )
+                .await?;
+                context.bob.write().await.status = 0; // secure-join failed
+                context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if !fingerprint_equals_sender(context, &scanned_fingerprint_of_alice, contact_chat_id) {
+            if !fingerprint_equals_sender(context, &scanned_fingerprint_of_alice, contact_chat_id)
+                .await?
+            {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
-                );
-                context.bob.write().unwrap().status = 0; // secure-join failed
-                context.stop_ongoing();
+                )
+                .await?;
+                context.bob.write().await.status = 0; // secure-join failed
+                context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
-            let own_fingerprint = get_self_fingerprint(context).unwrap();
+            let own_fingerprint = get_self_fingerprint(context).await.unwrap();
             joiner_progress!(context, contact_id, 400);
-            context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
+            context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -522,7 +564,8 @@ pub(crate) fn handle_securejoin_handshake(
                 } else {
                     "".to_string()
                 },
-            );
+            )
+            .await?;
             Ok(HandshakeMessage::Done)
         }
         "vg-request-with-auth" | "vc-request-with-auth" => {
@@ -533,31 +576,35 @@ pub(crate) fn handle_securejoin_handshake(
             ==========================================================*/
 
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
                         contact_chat_id,
                         "Fingerprint not provided.",
-                    );
+                    )
+                    .await?;
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !encrypted_and_signed(context, mime_message, &fingerprint) {
+            if !encrypted_and_signed(context, mime_message, Some(&fingerprint)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Auth not encrypted.",
-                );
+                )
+                .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if !fingerprint_equals_sender(context, &fingerprint, contact_chat_id) {
+            if !fingerprint_equals_sender(context, &fingerprint, contact_chat_id).await? {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
-                );
+                )
+                .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
@@ -569,25 +616,28 @@ pub(crate) fn handle_securejoin_handshake(
                         context,
                         contact_chat_id,
                         "Auth not provided.",
-                    );
+                    )
+                    .await?;
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !token::exists(context, token::Namespace::Auth, &auth_0) {
-                could_not_establish_secure_connection(context, contact_chat_id, "Auth invalid.");
+            if !token::exists(context, token::Namespace::Auth, &auth_0).await {
+                could_not_establish_secure_connection(context, contact_chat_id, "Auth invalid.")
+                    .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if mark_peer_as_verified(context, fingerprint).is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
-                );
+                )
+                .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited);
+            Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await;
             info!(context, "Auth verified.",);
-            secure_connection_established(context, contact_chat_id);
+            secure_connection_established(context, contact_chat_id).await?;
             emit_event!(context, Event::ContactsChanged(Some(contact_id)));
             inviter_progress!(context, contact_id, 600);
             if join_vg {
@@ -601,10 +651,11 @@ pub(crate) fn handle_securejoin_handshake(
                         return Ok(HandshakeMessage::Ignore);
                     }
                 };
-                match chat::get_chat_id_by_grpid(context, field_grpid) {
+                match chat::get_chat_id_by_grpid(context, field_grpid).await {
                     Ok((group_chat_id, _, _)) => {
                         if let Err(err) =
                             chat::add_contact_to_chat_ex(context, group_chat_id, contact_id, true)
+                                .await
                         {
                             error!(context, "failed to add contact: {}", err);
                         }
@@ -623,9 +674,11 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "vc-contact-confirm",
                     "",
-                    Some(fingerprint.clone()),
+                    Some(fingerprint),
                     "",
-                );
+                )
+                .await?;
+
                 inviter_progress!(context, contact_id, 1000);
             }
             Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed)
@@ -641,12 +694,12 @@ pub(crate) fn handle_securejoin_handshake(
                 HandshakeMessage::Ignore
             };
 
-            if context.bob.read().unwrap().expects != DC_VC_CONTACT_CONFIRM {
+            if context.bob.read().await.expects != DC_VC_CONTACT_CONFIRM {
                 info!(context, "Message belongs to a different handshake.",);
                 return Ok(abort_retval);
             }
             let cond = {
-                let bob = context.bob.read().unwrap();
+                let bob = context.bob.read().await;
                 let scan = bob.qr_scan.as_ref();
                 scan.is_none() || (join_vg && scan.unwrap().state != LotState::QrAskVerifyGroup)
             };
@@ -657,7 +710,8 @@ pub(crate) fn handle_securejoin_handshake(
                 );
                 return Ok(abort_retval);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
@@ -667,6 +721,7 @@ pub(crate) fn handle_securejoin_handshake(
                 // only after we have returned.  It does not impact
                 // the security invariants of secure-join however.
                 let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, &group_id)
+                    .await
                     .unwrap_or((ChatId::new(0), false, Blocked::Not));
                 // when joining a non-verified group
                 // the vg-member-added message may be unencrypted
@@ -678,26 +733,31 @@ pub(crate) fn handle_securejoin_handshake(
                 true
             };
             if vg_expect_encrypted
-                && !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice)
+                && !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice))
             {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Contact confirm message not encrypted.",
-                );
-                context.bob.write().unwrap().status = 0;
+                )
+                .await?;
+                context.bob.write().await.status = 0;
                 return Ok(abort_retval);
             }
 
-            if mark_peer_as_verified(context, &scanned_fingerprint_of_alice).is_err() {
+            if mark_peer_as_verified(context, &scanned_fingerprint_of_alice)
+                .await
+                .is_err()
+            {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
-                );
+                )
+                .await?;
                 return Ok(abort_retval);
             }
-            Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined);
+            Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined).await;
             emit_event!(context, Event::ContactsChanged(None));
             let cg_member_added = mime_message
                 .get(HeaderDef::ChatGroupMemberAdded)
@@ -706,13 +766,14 @@ pub(crate) fn handle_securejoin_handshake(
             if join_vg
                 && !context
                     .is_self_addr(cg_member_added)
+                    .await
                     .map_err(|_| HandshakeError::NoSelfAddr)?
             {
                 info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
                 return Ok(abort_retval);
             }
-            secure_connection_established(context, contact_chat_id);
-            context.bob.write().unwrap().expects = 0;
+            secure_connection_established(context, contact_chat_id).await?;
+            context.bob.write().await.expects = 0;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -726,10 +787,11 @@ pub(crate) fn handle_securejoin_handshake(
                 "",
                 Some(scanned_fingerprint_of_alice),
                 "",
-            );
+            )
+            .await?;
 
-            context.bob.write().unwrap().status = 1;
-            context.stop_ongoing();
+            context.bob.write().await.status = 1;
+            context.stop_ongoing().await;
             Ok(if join_vg {
                 HandshakeMessage::Propagate
             } else {
@@ -742,8 +804,8 @@ pub(crate) fn handle_securejoin_handshake(
             ====  Step 8 in "Out-of-band verified groups" protocol  ====
             ==========================================================*/
 
-            if let Ok(contact) = Contact::get_by_id(context, contact_id) {
-                if contact.is_verified(context) == VerifiedStatus::Unverified {
+            if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
+                if contact.is_verified(context).await == VerifiedStatus::Unverified {
                     warn!(context, "{} invalid.", step);
                     return Ok(HandshakeMessage::Ignore);
                 }
@@ -754,7 +816,7 @@ pub(crate) fn handle_securejoin_handshake(
                         .get(HeaderDef::SecureJoinGroup)
                         .map(|s| s.as_str())
                         .unwrap_or_else(|| "");
-                    if let Err(err) = chat::get_chat_id_by_grpid(context, &field_grpid) {
+                    if let Err(err) = chat::get_chat_id_by_grpid(context, &field_grpid).await {
                         warn!(context, "Failed to lookup chat_id from grpid: {}", err);
                         return Err(HandshakeError::ChatNotFound {
                             group: field_grpid.to_string(),
@@ -791,7 +853,7 @@ pub(crate) fn handle_securejoin_handshake(
 ///   the joining device has marked the peer as verified on vg-member-added/vc-contact-confirm
 ///   before sending vg-member-added-received - so, if we observe vg-member-added-received,
 ///   we can mark the peer as verified as well.
-pub(crate) fn observe_securejoin_on_other_device(
+pub(crate) async fn observe_securejoin_on_other_device(
     context: &Context,
     mime_message: &MimeMessage,
     contact_id: u32,
@@ -805,10 +867,10 @@ pub(crate) fn observe_securejoin_on_other_device(
     info!(context, "observing secure-join message \'{}\'", step);
 
     let contact_chat_id =
-        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not) {
+        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not).await {
             Ok((chat_id, blocked)) => {
                 if blocked != Blocked::Not {
-                    chat_id.unblock(context);
+                    chat_id.unblock(context).await;
                 }
                 chat_id
             }
@@ -828,32 +890,36 @@ pub(crate) fn observe_securejoin_on_other_device(
             if !encrypted_and_signed(
                 context,
                 mime_message,
-                get_self_fingerprint(context).unwrap_or_default(),
+                get_self_fingerprint(context).await.as_ref(),
             ) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Message not encrypted correctly.",
-                );
+                )
+                .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
                         contact_chat_id,
                         "Fingerprint not provided, please update Delta Chat on all your devices.",
-                    );
+                    )
+                    .await?;
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if mark_peer_as_verified(context, fingerprint).is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     format!("Fingerprint mismatch on observing {}.", step).as_ref(),
-                );
+                )
+                .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
             Ok(if step.as_str() == "vg-member-added" {
@@ -866,59 +932,69 @@ pub(crate) fn observe_securejoin_on_other_device(
     }
 }
 
-fn secure_connection_established(context: &Context, contact_chat_id: ChatId) {
-    let contact_id: u32 = chat_id_2_contact_id(context, contact_chat_id);
-    let contact = Contact::get_by_id(context, contact_id);
+async fn secure_connection_established(
+    context: &Context,
+    contact_chat_id: ChatId,
+) -> Result<(), HandshakeError> {
+    let contact_id: u32 = chat_id_2_contact_id(context, contact_chat_id).await?;
+    let contact = Contact::get_by_id(context, contact_id).await;
+
     let addr = if let Ok(ref contact) = contact {
         contact.get_addr()
     } else {
         "?"
     };
-    let msg = context.stock_string_repl_str(StockMessage::ContactVerified, addr);
-    chat::add_info_msg(context, contact_chat_id, msg);
+    let msg = context
+        .stock_string_repl_str(StockMessage::ContactVerified, addr)
+        .await;
+    chat::add_info_msg(context, contact_chat_id, msg).await;
     emit_event!(context, Event::ChatModified(contact_chat_id));
+
+    Ok(())
 }
 
-fn could_not_establish_secure_connection(
+async fn could_not_establish_secure_connection(
     context: &Context,
     contact_chat_id: ChatId,
     details: &str,
-) {
-    let contact_id = chat_id_2_contact_id(context, contact_chat_id);
-    let contact = Contact::get_by_id(context, contact_id);
-    let msg = context.stock_string_repl_str(
-        StockMessage::ContactNotVerified,
-        if let Ok(ref contact) = contact {
-            contact.get_addr()
-        } else {
-            "?"
-        },
-    );
+) -> Result<(), HandshakeError> {
+    let contact_id = chat_id_2_contact_id(context, contact_chat_id).await?;
+    let contact = Contact::get_by_id(context, contact_id).await;
+    let msg = context
+        .stock_string_repl_str(
+            StockMessage::ContactNotVerified,
+            if let Ok(ref contact) = contact {
+                contact.get_addr()
+            } else {
+                "?"
+            },
+        )
+        .await;
 
-    chat::add_info_msg(context, contact_chat_id, &msg);
+    chat::add_info_msg(context, contact_chat_id, &msg).await;
     error!(context, "{} ({})", &msg, details);
+    Ok(())
 }
 
-fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Result<(), Error> {
-    if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint.as_ref())
-    {
+async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
+    if let Ok(ref mut peerstate) = Peerstate::from_fingerprint(context, fingerprint).await {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
-            fingerprint.as_ref(),
+            fingerprint,
             PeerstateVerifiedStatus::BidirectVerified,
         ) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
             peerstate.to_save = Some(ToSave::All);
             peerstate
                 .save_to_db(&context.sql, false)
+                .await
                 .unwrap_or_default();
             return Ok(());
         }
     }
     bail!(
         "could not mark peer as verified for fingerprint {}",
-        fingerprint.as_ref()
+        fingerprint.hex()
     );
 }
 
@@ -929,7 +1005,7 @@ fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Res
 fn encrypted_and_signed(
     context: &Context,
     mimeparser: &MimeMessage,
-    expected_fingerprint: impl AsRef<str>,
+    expected_fingerprint: Option<&Fingerprint>,
 ) -> bool {
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
@@ -937,17 +1013,17 @@ fn encrypted_and_signed(
     } else if mimeparser.signatures.is_empty() {
         warn!(context, "Message not signed.",);
         false
-    } else if expected_fingerprint.as_ref().is_empty() {
-        warn!(context, "Fingerprint for comparison missing.",);
+    } else if expected_fingerprint.is_none() {
+        warn!(context, "Fingerprint for comparison missing.");
         false
     } else if !mimeparser
         .signatures
-        .contains(expected_fingerprint.as_ref())
+        .contains(expected_fingerprint.unwrap())
     {
         warn!(
             context,
             "Message does not match expected fingerprint {}.",
-            expected_fingerprint.as_ref(),
+            expected_fingerprint.unwrap(),
         );
         false
     } else {
@@ -955,33 +1031,32 @@ fn encrypted_and_signed(
     }
 }
 
-pub fn handle_degrade_event(context: &Context, peerstate: &Peerstate) -> Result<(), Error> {
+pub async fn handle_degrade_event(context: &Context, peerstate: &Peerstate) -> Result<(), Error> {
     // - we do not issue an warning for DC_DE_ENCRYPTION_PAUSED as this is quite normal
     // - currently, we do not issue an extra warning for DC_DE_VERIFICATION_LOST - this always comes
     //   together with DC_DE_FINGERPRINT_CHANGED which is logged, the idea is not to bother
     //   with things they cannot fix, so the user is just kicked from the verified group
     //   (and he will know this and can fix this)
     if Some(DegradeEvent::FingerprintChanged) == peerstate.degrade_event {
-        let contact_id: i32 = match context.sql.query_get_value(
-            context,
-            "SELECT id FROM contacts WHERE addr=?;",
-            params![&peerstate.addr],
-        ) {
-            None => bail!(
-                "contact with peerstate.addr {:?} not found",
-                &peerstate.addr
-            ),
-            Some(contact_id) => contact_id,
-        };
+        let contact_id: i32 = context
+            .sql
+            .query_value(
+                "SELECT id FROM contacts WHERE addr=?;",
+                paramsx![&peerstate.addr],
+            )
+            .await?;
+
         if contact_id > 0 {
             let (contact_chat_id, _) =
                 chat::create_or_lookup_by_contact_id(context, contact_id as u32, Blocked::Deaddrop)
+                    .await
                     .unwrap_or_default();
 
             let msg = context
-                .stock_string_repl_str(StockMessage::ContactSetupChanged, peerstate.addr.clone());
+                .stock_string_repl_str(StockMessage::ContactSetupChanged, peerstate.addr.clone())
+                .await;
 
-            chat::add_info_msg(context, contact_chat_id, msg);
+            chat::add_info_msg(context, contact_chat_id, msg).await;
             emit_event!(context, Event::ChatModified(contact_chat_id));
         }
     }

src/smtp/mod.rs

@@ -1,5 +1,7 @@
 //! # SMTP transport module
 
+#![forbid(clippy::indexing_slicing)]
+
 pub mod send;
 
 use std::time::{Duration, Instant};
@@ -10,8 +12,10 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
-use crate::login_param::{dc_build_tls, LoginParam};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::oauth2::*;
+use crate::provider::get_provider_info;
+use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
@@ -43,9 +47,8 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Default, DebugStub)]
-pub struct Smtp {
-    #[debug_stub(some = "SmtpTransport")]
+#[derive(Default)]
+pub(crate) struct Smtp {
     transport: Option<smtp::SmtpTransport>,
 
     /// Email address we are sending from.
@@ -64,16 +67,16 @@ impl Smtp {
     }
 
     /// Disconnect the SMTP transport and drop it entirely.
-    pub fn disconnect(&mut self) {
+    pub async fn disconnect(&mut self) {
         if let Some(mut transport) = self.transport.take() {
-            async_std::task::block_on(transport.close()).ok();
+            transport.close().await.ok();
         }
         self.last_success = None;
     }
 
     /// Return true if smtp was connected but is not known to
     /// have been successfully used the last 60 seconds
-    pub fn has_maybe_stale_connection(&self) -> bool {
+    pub async fn has_maybe_stale_connection(&self) -> bool {
         if let Some(last_success) = self.last_success {
             Instant::now().duration_since(last_success).as_secs() > 60
         } else {
@@ -82,7 +85,7 @@ impl Smtp {
     }
 
     /// Check whether we are connected.
-    pub fn is_connected(&self) -> bool {
+    pub async fn is_connected(&self) -> bool {
         self.transport
             .as_ref()
             .map(|t| t.is_connected())
@@ -90,18 +93,14 @@ impl Smtp {
     }
 
     /// Connect using the provided login params.
-    pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
-        async_std::task::block_on(self.inner_connect(context, lp))
-    }
-
-    async fn inner_connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
-        if self.is_connected() {
+    pub async fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
+        if self.is_connected().await {
             warn!(context, "SMTP already connected.");
             return Ok(());
         }
 
         if lp.send_server.is_empty() || lp.send_port == 0 {
-            context.call_cb(Event::ErrorNetwork("SMTP bad parameters.".into()));
+            context.emit_event(Event::ErrorNetwork("SMTP bad parameters.".into()));
             return Err(Error::BadParameters);
         }
 
@@ -110,19 +109,27 @@ impl Smtp {
                 address: lp.addr.clone(),
                 error: err,
             })?;
+
         self.from = Some(from);
 
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls_config = dc_build_tls(lp.smtp_certificate_checks);
+        let provider = get_provider_info(&lp.addr);
+        let strict_tls = match lp.smtp_certificate_checks {
+            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Strict => true,
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => false,
+        };
+        let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
             // oauth2
             let addr = &lp.addr;
             let send_pw = &lp.send_pw;
-            let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false);
+            let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false).await;
             if access_token.is_none() {
                 return Err(Error::Oauth2Error {
                     address: addr.to_string(),
@@ -169,15 +176,23 @@ impl Smtp {
             .timeout(Some(Duration::from_secs(SMTP_TIMEOUT)));
 
         let mut trans = client.into_transport();
+        if let Err(err) = trans.connect().await {
+            let message = context
+                .stock_string_repl_str2(
+                    StockMessage::ServerResponse,
+                    format!("SMTP {}:{}", domain, port),
+                    format!("{}, ({:?})", err.to_string(), err),
+                )
+                .await;
 
-        trans.connect().await.map_err(|err| {
-            emit_event!(context, Event::ErrorNetwork(err.to_string()));
-            Error::ConnectionFailure(err)
-        })?;
+            emit_event!(context, Event::ErrorNetwork(message));
+            return Err(Error::ConnectionFailure(err));
+        }
 
         self.transport = Some(trans);
         self.last_success = Some(Instant::now());
-        context.call_cb(Event::SmtpConnected(format!(
+
+        context.emit_event(Event::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
             lp.send_user,
         )));

src/smtp/send.rs

@@ -49,7 +49,7 @@ impl Smtp {
         if let Some(ref mut transport) = self.transport {
             transport.send(mail).await.map_err(Error::SendError)?;
 
-            context.call_cb(Event::SmtpMessageSent(format!(
+            context.emit_event(Event::SmtpMessageSent(format!(
                 "Message len={} was smtp-sent to {}",
                 message_len, recipients_display
             )));