Download & decrypt uploaded attachements.

- Parse the "Chat-Upload-Url" header
- Add message::schedule_download function to schedule download job
- In the job download, decrypt, save

.circleci/config.yml

@@ -138,12 +138,6 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests_rust:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_rust.sh
-    
   remote_tests_python:
     machine: true
     steps:
@@ -178,11 +172,6 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
-
       - remote_tests_python:
           filters:
             tags:
@@ -191,8 +180,9 @@ workflows:
       - remote_python_packaging:
           requires: 
              - remote_tests_python
-             - remote_tests_rust 
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 
@@ -201,6 +191,8 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -212,6 +204,8 @@ workflows:
 
       - build_doxygen:
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: Rust CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.43.1
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.43.1
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly, 1.43.1]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      with:
+        command:  check
+        args: --workspace --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --workspace
+          

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-19
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

CHANGELOG.md

@@ -1,5 +1,44 @@
 # Changelog 
 
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
+## 1.34.0
+
+- new api for io, thread and event handling #1356,
+  see the example atop of `deltachat.h` to get an overview
+
+- LOTS of speed improvements due to async processing #1356
+
+- enable WAL mode for sqlite #1492
+
+- process incoming messages in bulk #1527
+
+- improve finding out the sent-folder #1488
+
+- several bug fixes
+
+
 ## 1.33.0
 
 - let `dc_set_muted()` also mute one-to-one chats #1470

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.33.0"
+version = "1.35.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,21 +12,21 @@ license = "MPL-2.0"
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
 async-smtp = { version = "0.3" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.3.0"
+async-imap = "0.3.1"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.0", features = ["unstable"] }
-base64 = "0.11"
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
@@ -35,43 +35,44 @@ chrono = "0.4.6"
 indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.22", features = ["bundled"] }
-r2d2_sqlite = "0.15.0"
+rusqlite = { version = "0.23", features = ["bundled"] }
+r2d2_sqlite = "0.16.0"
 r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
+strum = "0.18.0"
+strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.17.1"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.0"
+mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
 
-pretty_env_logger = { version = "0.3.1", optional = true }
+pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+open = { version = "1.4.0", optional = true }
 
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
 async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+smol = "0.1.10"
 
 [workspace]
 members = [
@@ -82,6 +83,7 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
+required-features = ["repl"]
 
 [[example]]
 name = "repl"
@@ -92,7 +94,7 @@ required-features = ["repl"]
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "open"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 

README.md

@@ -132,4 +132,5 @@ or its language bindings:
 - [iOS](https://github.com/deltachat/deltachat-ios)
 - [Desktop](https://github.com/deltachat/deltachat-desktop)
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - several **Bots**

appveyor.yml

@@ -1,19 +0,0 @@
-environment:
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-19
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - rustc -vV
-  - cargo -vV
-
-build: false
-
-test_script:
-  - cargo test --release --all
-
-cache:
-  - target
-  - C:\Users\appveyor\.cargo\registry

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.33.0"
+version = "1.35.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -32,63 +32,40 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  *
  * Let's start.
  *
- * First of all, you have to **define an event-handler-function**
- * that is called by the library on specific events
- * (eg. when the configuration is done or when fresh messages arrive).
- * With this function you can create a Delta Chat context then:
+ * First of all, you have to **create a context object**
+ * bound to a database.
+ * The database is a normal sqlite-file and is created as needed:
  *
  * ~~~
- * #include <deltachat.h>
+ * dc_context_t* context = dc_context_new(NULL, "example.db", NULL);
+ * ~~~
  *
- * uintptr_t event_handler_func(dc_context_t* context, int event,
- *                              uintptr_t data1, uintptr_t data2)
+ * After that, make sure, you can **receive events from the context**.
+ * For that purpose, create an event emitter you can ask for events.
+ * If there are no event, the emitter will wait until there is one,
+ * so, in many situations you will do this in a thread:
+ *
+ * ~~~
+ * void* event_handler(void* context)
  * {
- *     return 0;
- * }
- *
- * dc_context_t* context = dc_context_new(event_handler_func, NULL, NULL);
- * ~~~
- *
- * After that, you should make sure,
- * sending and receiving jobs are processed as needed.
- * For this purpose, you have to **create two threads:**
- *
- * ~~~
- * #include <pthread.h>
- *
- * void* imap_thread_func(void* context)
- * {
- *     while (true) {
- *         dc_perform_imap_jobs(context);
- *         dc_perform_imap_fetch(context);
- *         dc_perform_imap_idle(context);
+ *     dc_event_emitter_t* emitter = dc_get_event_emitter(context);
+ *     dc_event_t* event;
+ *     while ((event = dc_get_next_event(emitter)) != NULL) {
+ *         // use the event as needed, eg. dc_event_get_id() returns the type.
+ *         // once you're done, unref the event to avoid memory leakage:
+ *         dc_event_unref(event);
  *     }
+ *     dc_event_emitter_unref(emitter);
  * }
  *
- * void* smtp_thread_func(void* context)
- * {
- *     while (true) {
- *         dc_perform_smtp_jobs(context);
- *         dc_perform_smtp_idle(context);
- *     }
- * }
- *
- * static pthread_t imap_thread, smtp_thread;
- * pthread_create(&imap_thread, NULL, imap_thread_func, context);
- * pthread_create(&smtp_thread, NULL, smtp_thread_func, context);
+ * static pthread_t event_thread;
+ * pthread_create(&event_thread, NULL, event_handler, context);
  * ~~~
  *
  * The example above uses "pthreads",
  * however, you can also use anything else for thread handling.
  * All deltachat-core-functions, unless stated otherwise, are thread-safe.
  *
- * After that you can  **define and open a database.**
- * The database is a normal sqlite-file and is created as needed:
- *
- * ~~~
- * dc_open(context, "example.db", NULL);
- * ~~~
- *
  * Now you can **configure the context:**
  *
  * ~~~
@@ -98,15 +75,22 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  * dc_configure(context);
  * ~~~
  *
- * dc_configure() returns immediately, the configuration itself may take a while
- * and is done by a job in the imap-thread you've defined above.
+ * dc_configure() returns immediately,
+ * the configuration itself runs in background and may take a while.
  * Once done, the #DC_EVENT_CONFIGURE_PROGRESS reports success
- * to the event_handler_func() that is also defined above.
+ * to the event_handler() you've defined above.
  *
  * The configuration result is saved in the database,
  * on subsequent starts it is not needed to call dc_configure()
  * (you can check this using dc_is_configured()).
  *
+ * On a successfully configured context,
+ * you can finally **connect to the servers:**
+ *
+ * ~~~
+ * dc_start_io(context);
+ * ~~~
+ *
  * Now you can **send the first message:**
  *
  * ~~~
@@ -118,11 +102,11 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  * ~~~
  *
  * dc_send_text_msg() returns immediately;
- * the sending itself is done by a job in the smtp-thread you've defined above.
+ * the sending itself is done in the background.
  * If you check the testing address (bob)
  * and you should have received a normal email.
  * Answer this email in any email program with "Got it!"
- * and the imap-thread you've create above will **receive the message**.
+ * and the IO you started above will **receive the message**.
  *
  * You can then **list all messages** of a chat as follow:
  *
@@ -166,19 +150,6 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  * - The issue-tracker for the core library is here:
  *   <https://github.com/deltachat/deltachat-core-rust/issues>
  *
- * The following points are important mainly
- * for the authors of the library itself:
- *
- * - For indentation, use tabs.
- *   Alignments that are not placed at the beginning of a line
- *   should be done with spaces.
- *
- * - For padding between functions,
- *   classes etc. use 2 empty lines
- *
- * - Source files are encoded as UTF-8 with Unix line endings
- *   (a simple `LF`, `0x0A` or `\n`)
- *
  * If you need further assistance,
  * please do not hesitate to contact us
  * through the channels shown at https://delta.chat/en/contribute
@@ -191,24 +162,6 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  */
 
 
-/**
- * TODO: document
- */  
-dc_event_t*   dc_get_next_event(dc_event_emitter_t* emitter);
-
-dc_event_emitter_t* dc_get_event_emitter(dc_context_t* context);
-void  dc_event_emitter_unref(dc_event_emitter_t* emitter);
-
-int           dc_event_get_id   (dc_event_t* event);
-int           dc_event_get_data1_int(dc_event_t* event);
-int           dc_event_get_data2_int(dc_event_t* event);
-char*         dc_event_get_data3_str(dc_event_t* event);
-
-/**
- * TODO: document
- */
-void          dc_event_unref   (dc_event_t* event);
-
 /**
  * @class dc_context_t
  *
@@ -226,21 +179,6 @@ void          dc_event_unref   (dc_event_t* event);
  * opened, connected and mails are fetched.
  *
  * @memberof dc_context_t
- * @param cb a callback function that is called for events (update,
- *     state changes etc.) and to get some information from the client (eg. translation
- *     for a given string).
- *     See @ref DC_EVENT for a list of possible events that may be passed to the callback.
- *     - The callback MAY be called from _any_ thread, not only the main/GUI thread!
- *     - The callback MUST NOT call any dc_* and related functions unless stated
- *       otherwise!
- *     - The callback SHOULD return _fast_, for GUI updates etc. you should
- *       post yourself an asynchronous message to your GUI thread, if needed.
- *     - events do not expect a return value, just always return 0.
- * @param dbfile The file to use to store the database, something like `~/file` won't
- *     work on all systems, if in doubt, use absolute paths.
- * @param blobdir A directory to store the blobs in; a trailing slash is not needed.
- *     If you pass NULL or the empty string, deltachat-core creates a directory
- *     beside _dbfile_ with the same name and the suffix `-blobs`.
  * @param os_name is only for decorative use
  *     and is shown eg. in the `X-Mailer:` header
  *     in the form "Delta Chat Core <version>/<os_name>".
@@ -248,6 +186,11 @@ void          dc_event_unref   (dc_event_t* event);
  *     the used environment and/or the version here.
  *     It is okay to give NULL, in this case `X-Mailer:` header
  *     is set to "Delta Chat Core <version>".
+ * @param dbfile The file to use to store the database,
+ *     something like `~/file` won't work, use absolute paths.
+ * @param blobdir A directory to store the blobs in; a trailing slash is not needed.
+ *     If you pass NULL or the empty string, deltachat-core creates a directory
+ *     beside _dbfile_ with the same name and the suffix `-blobs`.
  * @return A context object with some public members.
  *     The object must be passed to the other context functions
  *     and must be freed using dc_context_unref() after usage.
@@ -269,6 +212,25 @@ dc_context_t*   dc_context_new               (const char* os_name, const char* d
  */
 void            dc_context_unref             (dc_context_t* context);
 
+
+/**
+ * Create the event emitter that is used to receive events.
+ * The library will emit various @ref DC_EVENT events as "new message", "message read" etc.
+ * To get these events, you have to create an event emitter using this function
+ * and call dc_get_next_event() on the emitter.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as created by dc_context_new().
+ * @return Returns the event emitter, NULL on errors.
+ *     Must be freed using dc_event_emitter_unref() after usage.
+ *
+ * Note: Use only one event emitter per context.
+ * Having more than one event emitter running at the same time on the same context
+ * will result in events randomly delivered to the one or to the other.
+ */
+dc_event_emitter_t* dc_get_event_emitter(dc_context_t* context);
+
+
 /**
  * Get the blob directory.
  *
@@ -468,9 +430,7 @@ char*           dc_get_oauth2_url            (dc_context_t* context, const char*
 
 /**
  * Configure a context.
- * For this purpose, the function creates a job
- * that is executed in the IMAP-thread then;
- * this requires to call dc_perform_imap_jobs() regularly.
+ * While configuration IO must not be started, if needed stop IO using dc_stop_io() first.
  * If the context is already configured,
  * this function will try to change the configuration.
  *
@@ -537,6 +497,8 @@ int             dc_is_configured   (const dc_context_t* context);
 
 /**
  * Start job and IMAP/SMTP tasks.
+ * If IO is already running, nothing happens.
+ * To check the current IO state, use dc_is_io_running().
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
@@ -556,6 +518,8 @@ int             dc_is_io_running(const dc_context_t* context);
 
 /**
  * Stop job and IMAP/SMTP tasks and return when they are finished. 
+ * If IO is not running, nothing happens.
+ * To check the current IO state, use dc_is_io_running().
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
@@ -1674,9 +1638,6 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
 
 /**
  * Import/export things.
- * For this purpose, the function creates a job that is executed in the IMAP-thread then;
- * this requires to call dc_perform_imap_jobs() regularly.
- *
  * What to do is defined by the _what_ parameter which may be one of the following:
  *
  * - **DC_IMEX_EXPORT_BACKUP** (11) - Export a backup to the directory given as `param1`.
@@ -3202,6 +3163,54 @@ int             dc_msg_is_setupmessage        (const dc_msg_t* msg);
 char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
 
 
+/**
+ * Check if the message is completely downloaded
+ * or if some further action is needed.
+ *
+ * The function returns one of:
+ * - @ref DC_DOWNLOAD_NO_URL      - The message does not need any further download action
+ *                                  and should be rendered as usual.
+ * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
+ *                                  Tn addition to the usual message rendering,
+ *                                  the UI shall show a download button that starts dc_schedule_download()
+ * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_schedule_download() and is still in progress.
+ *                                  On progress changes and if the download fails or succeeds,
+ *                                  the event @ref DC_EVENT_DOWNLOAD_PROGRESS will be emitted.
+ * - @ref DC_DOWNLOAD_DONE        - Download finished successfully
+ * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_schedule_download() again.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return One of the @ref DC_DOWNLOAD values
+ */
+int dc_msg_download_status(const dc_msg_t* msg);
+
+
+/**
+ * Advices the core to start downloading a message.
+ * This function is typically called when the user hits the "Download" button
+ * that is shown by the UI in case dc_msg_download_status()
+ * returns @ref DC_DOWNLOAD_AVAILABLE or @ref DC_DOWNLOAD_FAILURE.
+ *
+ * The UI may want to show a file selector and let the user chose a download location.
+ * The file name in the file selector may be prefilled using dc_msg_get_filename().
+ *
+ * During the download, the progress, errors and success
+ * are reported using @ref DC_EVENT_DOWNLOAD_PROGRESS.
+ *
+ * Once the @ref DC_EVENT_DOWNLOAD_PROGRESS reports success,
+ * The file can be accessed as usual using dc_msg_get_file().
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param path Path to the destination file.
+ *     You can specify NULL here to download
+ *     to a reasonable file name in the internal blob-directory.
+ * @param msg_id Message-ID to download the content for.
+ */
+void dc_schedule_download(dc_context_t* context, int msg_id, const char* path);
+
+
 /**
  * Set the text of a message object.
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
@@ -3857,11 +3866,122 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EMPTY_INBOX 0x02 // Deprecated, flag for dc_empty_server(): Clear all INBOX messages
 
 
+/**
+ * @class dc_event_emitter_t
+ *
+ * Opaque object that is used to get events.
+ * You can get an event emitter from a context using dc_get_event_emitter().
+ */
+
+/**
+ * Get the next event from an event emitter object.
+ *
+ * @memberof dc_event_emitter_t
+ * @param emitter Event emitter object as returned from dc_get_event_emitter().
+ * @return An event as an dc_event_t object.
+ *     You can query the event for information using dc_event_get_id(), dc_event_get_data1_int() and so on;
+ *     if you are done with the event, you have to free the event using dc_event_unref().
+ *     If NULL is returned, the context belonging to the event emitter is unref'd and the no more events will come;
+ *     in this case, free the event emitter using dc_event_emitter_unref().
+ */
+dc_event_t* dc_get_next_event(dc_event_emitter_t* emitter);
+
+
+/**
+ * Free an event emitter object.
+ *
+ * @memberof dc_event_emitter_t
+ * @param emitter Event emitter object as returned from dc_get_event_emitter().
+ *     If NULL is given, nothing is done and an error is logged.
+ * @return None.
+ */
+void  dc_event_emitter_unref(dc_event_emitter_t* emitter);
+
+
+/**
+ * @class dc_event_t
+ *
+ * Opaque object describing a single event.
+ * To get events, call dc_get_next_event() on an event emitter created by dc_get_event_emitter().
+ */
+
+/**
+ * Get the event-id from an event object.
+ * The event-id is one of the @ref DC_EVENT constants.
+ * There may be additional data belonging to an event,
+ * to get them, use dc_event_get_data1_int(), dc_event_get_data2_int() and dc_event_get_data2_str().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return once of the @ref DC_EVENT constants.
+ *     0 on errors.
+ */
+int dc_event_get_id(dc_event_t* event);
+
+
+/**
+ * Get a data associated with an event object.
+ * The meaning of the data depends on the event-id
+ * returned as @ref DC_EVENT constants by dc_event_get_id().
+ * See also dc_event_get_data2_int() and dc_event_get_data2_str().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data1" as a signed integer, at least 32bit,
+ *     the meaning depends on the event type associated with this event.
+ */
+int dc_event_get_data1_int(dc_event_t* event);
+
+
+/**
+ * Get a data associated with an event object.
+ * The meaning of the data depends on the event-id
+ * returned as @ref DC_EVENT constants by dc_event_get_id().
+ * See also dc_event_get_data2_int() and dc_event_get_data2_str().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data2" as a signed integer, at least 32bit,
+ *     the meaning depends on the event type associated with this event.
+ */
+int dc_event_get_data2_int(dc_event_t* event);
+
+
+/**
+ * Get a data associated with an event object.
+ * The meaning of the data depends on the event-id
+ * returned as @ref DC_EVENT constants by dc_event_get_id().
+ * See also dc_event_get_data1_int() and dc_event_get_data2_int().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data2" as a string,
+ *     the meaning depends on the event type associated with this event.
+ *     Once you're done with the string, you have to unref it using dc_unref_str().
+ */
+char* dc_event_get_data2_str(dc_event_t* event);
+
+
+/**
+ * Free memory used by an event object.
+ * If you forget to do this for an event, this will result in memory leakage.
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return None.
+ */
+void dc_event_unref(dc_event_t* event);
+
+
 /**
  * @defgroup DC_EVENT DC_EVENT
  *
- * These constants are used as events
- * reported to the callback given to dc_context_new().
+ * These constants are used as event-id
+ * in events returned by dc_get_next_event().
+ *
+ * Events typically come with some additional data,
+ * use dc_event_get_data1_int(), dc_event_get_data2_int() and dc_event_get_data2_str() to read this data.
+ * The meaning of the data depends on the event.
  *
  * @addtogroup DC_EVENT
  * @{
@@ -3869,13 +3989,11 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * The library-user may write an informational string to the log.
- * Passed to the callback given to dc_context_new().
  *
  * This event should not be reported to the end-user using a popup or something like that.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_INFO                     100
 
@@ -3884,8 +4002,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when SMTP connection is established and login was successful.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_SMTP_CONNECTED           101
 
@@ -3894,8 +4011,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when IMAP connection is established and login was successful.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_IMAP_CONNECTED           102
 
@@ -3903,8 +4019,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a message was successfully sent to the SMTP server.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_SMTP_MESSAGE_SENT        103
 
@@ -3912,8 +4027,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a message was successfully marked as deleted on the IMAP server.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_IMAP_MESSAGE_DELETED   104
 
@@ -3921,8 +4035,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a message was successfully moved on IMAP.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
@@ -3930,8 +4043,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when an IMAP folder was emptied.
  *
  * @param data1 0
- * @param data2 (const char*) folder name.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Folder name.
  */
 #define DC_EVENT_IMAP_FOLDER_EMPTIED  106
 
@@ -3939,8 +4051,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a new blob file was successfully written
  *
  * @param data1 0
- * @param data2 (const char*) path name
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Path name
  */
 #define DC_EVENT_NEW_BLOB_FILE 150
 
@@ -3948,27 +4059,23 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a blob file was successfully deleted
  *
  * @param data1 0
- * @param data2 (const char*) path name
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Path name
  */
 #define DC_EVENT_DELETED_BLOB_FILE 151
 
 /**
  * The library-user should write a warning string to the log.
- * Passed to the callback given to dc_context_new().
  *
  * This event should not be reported to the end-user using a popup or something like that.
  *
  * @param data1 0
- * @param data2 (const char*) Warning string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Warning string in english language.
  */
 #define DC_EVENT_WARNING                  300
 
 
 /**
  * The library-user should report an error to the end-user.
- * Passed to the callback given to dc_context_new().
  *
  * As most things are asynchronous, things may go wrong at any time and the user
  * should not be disturbed by a dialog or so.  Instead, use a bubble or so.
@@ -3980,10 +4087,9 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * in a messasge box then.
  *
  * @param data1 0
- * @param data2 (const char*) Error string, always set, never NULL.
+ * @param data2 (char*) Error string, always set, never NULL.
  *     Some error strings are taken from dc_set_stock_translation(),
  *     however, most error strings will be in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
  */
 #define DC_EVENT_ERROR                    400
 
@@ -4005,8 +4111,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 1=first/new network error, should be reported the user;
  *     0=subsequent network error, should be logged only
- * @param data2 (const char*) Error string, always set, never NULL.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Error string, always set, never NULL.
  */
 #define DC_EVENT_ERROR_NETWORK            401
 
@@ -4019,9 +4124,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * dc_send_text_msg() or another sending function.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified
- *     and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_ERROR_SELF_NOT_IN_GROUP  410
 
@@ -4139,9 +4242,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * A typical purpose for a handler of this event may be to make the file public to some system
  * services.
  *
- * @param data1 (const char*) Path and file name.
- *     Must not be unref'd or modified and is valid only until the callback returns.
- * @param data2 0
+ * @param data1 0
+ * @param data2 (char*) Path and file name.
  */
 #define DC_EVENT_IMEX_FILE_WRITTEN        2052
 
@@ -4177,6 +4279,16 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
+
+/**
+ * Inform about the progress of a download started by dc_schedule_download().
+ *
+ * @param data1 (int) Message-ID the progress is reported for.
+ * @param data2 (int) 0=error, 1-999=progress in permille, 1000=success and done
+ */
+#define DC_EVENT_DOWNLOAD_PROGRESS       2070
+
+
 /**
  * @}
  */
@@ -4316,6 +4428,29 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
+/**
+ * @defgroup DC_DOWNLOAD DC_DOWNLOAD
+ *
+ * These constants describe the download state of a message.
+ * The download state can be retrieved using dc_msg_download_status()
+ * and usually changes after calling dc_schedule_download().
+ *
+ * @addtogroup DC_DOWNLOAD
+ * @{
+ */
+
+#define DC_DOWNLOAD_NO_URL      10 ///< Download not needed, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_AVAILABLE   20 ///< Download available, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_IN_PROGRESS 30 ///< Download in progress, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_DONE        40 ///< Download done, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_FAILURE     50 ///< Download failed, see dc_msg_download_status() for details.
+
+/**
+ * @}
+ */
+
+
+
 /*
  * TODO: Strings need some doumentation about used placeholders.
  *
@@ -4369,12 +4504,24 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_STR_LOCATION                   66
 #define DC_STR_STICKER                    67
 #define DC_STR_DEVICE_MESSAGES            68
-#define DC_STR_COUNT                      68
+#define DC_STR_SAVED_MESSAGES             69
+#define DC_STR_DEVICE_MESSAGES_HINT       70
+#define DC_STR_WELCOME_MESSAGE            71
+#define DC_STR_UNKNOWN_SENDER_FOR_CHAT    72
+#define DC_STR_SUBJECT_FOR_NEW_CONTACT    73
+#define DC_STR_COUNT                      73
 
 /*
  * @}
  */
 
+#ifdef PY_CFFI_INC
+/* Helper utility to locate the header file when building python bindings. */
+char* _dc_header_file_location(void) {
+    return __FILE__;
+}
+#endif
+
 
 #ifdef __cplusplus
 }

deltachat-ffi/src/lib.rs

@@ -403,9 +403,9 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_event_get_data3_str(event: *mut dc_event_t) -> *mut libc::c_char {
+pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut libc::c_char {
     if event.is_null() {
-        eprintln!("ignoring careless call to dc_event_get_data3_str()");
+        eprintln!("ignoring careless call to dc_event_get_data2_str()");
         return ptr::null_mut();
     }
 

examples/repl/cmdline.rs

@@ -370,6 +370,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 openfile <msg-id>\n\
+                 download <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
@@ -890,6 +892,25 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
+        "openfile" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let msg = Message::load_from_db(&context, id).await?;
+            let filepath = msg.get_file(&context);
+            ensure!(filepath.is_some(), "Message has no file.");
+            let filepath = filepath.unwrap();
+            open::that(filepath)?;
+        }
+        "download" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let path = if !arg2.is_empty() {
+                Some(arg2.into())
+            } else {
+                None
+            };
+            message::schedule_download(&context, id, path).await?;
+        }
         "listfresh" => {
             let msglist = context.get_fresh_msgs().await;
 

examples/repl/main.rs

@@ -146,10 +146,8 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 11] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
-    "open",
-    "close",
     "set",
     "get",
     "oauth2",
@@ -188,9 +186,11 @@ const CHAT_COMMANDS: [&str; 26] = [
     "unpin",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 10] = [
     "listmsgs",
     "msginfo",
+    "openfile",
+    "download",
     "listfresh",
     "forward",
     "markseen",
@@ -290,48 +290,59 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let h = DcHelper {
-        completer: FilenameCompleter::new(),
-        highlighter: MatchingBracketHighlighter::new(),
-        hinter: HistoryHinter {},
-    };
-    let mut rl = Editor::with_config(config);
-    rl.set_helper(Some(h));
-    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
-    if rl.load_history(".dc-history.txt").is_err() {
-        println!("No previous history.");
-    }
-
     let mut selected_chat = ChatId::default();
+    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let input_loop = async_std::task::spawn_blocking(move || {
+        let h = DcHelper {
+            completer: FilenameCompleter::new(),
+            highlighter: MatchingBracketHighlighter::new(),
+            hinter: HistoryHinter {},
+        };
+        let mut rl = Editor::with_config(config);
+        rl.set_helper(Some(h));
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        if rl.load_history(".dc-history.txt").is_err() {
+            println!("No previous history.");
+        }
 
-    loop {
-        let p = "> ";
-        let readline = rl.readline(&p);
-        match readline {
-            Ok(line) => {
-                // TODO: ignore "set mail_pw"
-                rl.add_history_entry(line.as_str());
-                match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-                    Ok(ExitResult::Continue) => {}
-                    Ok(ExitResult::Exit) => break,
-                    Err(err) => println!("Error: {}", err),
+        loop {
+            let p = "> ";
+            let readline = rl.readline(&p);
+
+            match readline {
+                Ok(line) => {
+                    // TODO: ignore "set mail_pw"
+                    rl.add_history_entry(line.as_str());
+                    async_std::task::block_on(reader_s.send(line));
+                }
+                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                    println!("Exiting...");
+                    drop(reader_s);
+                    break;
+                }
+                Err(err) => {
+                    println!("Error: {}", err);
+                    drop(reader_s);
+                    break;
                 }
             }
-            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-                println!("Exiting...");
-                context.stop_io().await;
-                break;
-            }
-            Err(err) => {
-                println!("Error: {}", err);
-                break;
-            }
+        }
+
+        rl.save_history(".dc-history.txt")?;
+        println!("history saved");
+        Ok::<_, Error>(())
+    });
+
+    while let Ok(line) = reader_r.recv().await {
+        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
+            Ok(ExitResult::Continue) => {}
+            Ok(ExitResult::Exit) => break,
+            Err(err) => println!("Error: {}", err),
         }
     }
-
-    rl.save_history(".dc-history.txt")?;
-    println!("history saved");
+    context.stop_io().await;
+    input_loop.await?;
 
     Ok(())
 }

examples/simple.rs

@@ -1,6 +1,3 @@
-extern crate deltachat;
-
-use std::time;
 use tempfile::tempdir;
 
 use deltachat::chat;
@@ -8,35 +5,42 @@ use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
+use deltachat::message::Message;
 use deltachat::Event;
 
 fn cb(event: Event) {
-    print!("[{:?}]", event);
-
     match event {
         Event::ConfigureProgress(progress) => {
-            println!("  progress: {}", progress);
+            log::info!("progress: {}", progress);
         }
-        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            println!("  {}", msg);
+        Event::Info(msg) => {
+            log::info!("{}", msg);
         }
-        _ => {
-            println!();
+        Event::Warning(msg) => {
+            log::warn!("{}", msg);
+        }
+        Event::Error(msg) | Event::ErrorNetwork(msg) => {
+            log::error!("{}", msg);
+        }
+        event => {
+            log::info!("{:?}", event);
         }
     }
 }
 
+/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
 #[async_std::main]
 async fn main() {
+    pretty_env_logger::try_init_timed().ok();
+
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    println!("creating database {:?}", dbfile);
+    log::info!("creating database {:?}", dbfile);
     let ctx = Context::new("FakeOs".into(), dbfile.into())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
-    let duration = time::Duration::from_millis(4000);
-    println!("info: {:#?}", info);
+    log::info!("info: {:#?}", info);
 
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
@@ -45,7 +49,7 @@ async fn main() {
         }
     });
 
-    println!("configuring");
+    log::info!("configuring");
     let args = std::env::args().collect::<Vec<String>>();
     assert_eq!(args.len(), 3, "requires email password");
     let email = args[1].clone();
@@ -59,35 +63,38 @@ async fn main() {
 
     ctx.configure().await.unwrap();
 
-    println!("------ RUN ------");
-    ctx.clone().start_io().await;
-    println!("--- SENDING A MESSAGE ---");
+    log::info!("------ RUN ------");
+    ctx.start_io().await;
+    log::info!("--- SENDING A MESSAGE ---");
 
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
     let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
-    for i in 0..2 {
+    for i in 0..1 {
+        log::info!("sending message {}", i);
         chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
             .await
             .unwrap();
     }
 
-    println!("fetching chats..");
+    // wait for the message to be sent out
+    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+
+    log::info!("fetching chats..");
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let summary = chats.get_summary(&ctx, 0, None).await;
-        let text1 = summary.get_text1();
-        let text2 = summary.get_text2();
-        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
+            .await
+            .unwrap();
+        log::info!("[{}] msg: {:?}", i, msg);
     }
 
-    async_std::task::sleep(duration).await;
-
-    println!("stopping");
+    log::info!("stopping");
     ctx.stop_io().await;
-    println!("closing");
+    log::info!("closing");
+    drop(ctx);
     events_spawn.await;
 }

python/examples/echo_and_quit.py

@@ -12,7 +12,7 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
                 message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))

python/examples/test_examples.py

@@ -26,15 +26,15 @@ def test_echo_quit_plugin(acfactory, lp):
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
+    bot_chat = bot_contact.create_chat()
+    bot_chat.send_text("hello")
 
-    lp.sec("waiting for the bot-reply to arrive")
+    lp.sec("waiting for the reply message from the bot to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    assert reply.chat == ch1
     lp.sec("send quit sequence")
-    ch1.send_text("/quit")
+    bot_chat.send_text("/quit")
     botproc.wait()
 
 
@@ -47,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+    ac1.add_account_plugin(FFIEventLogger(ac1))
+    ac2.add_account_plugin(FFIEventLogger(ac2))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -60,7 +60,8 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = events.FFIEventLogger(ac, "bot")
+        ac.set_config("displayname", "bot")
+        log = events.FFIEventLogger(ac)
         ac.add_account_plugin(log)
 
     for plugin in account_plugins or []:
@@ -76,8 +77,8 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.configure()
-        ac.wait_configure_finish()
+        configtracker = ac.configure()
+        configtracker.wait_finish()
 
     # start IO threads and configure if neccessary
     ac.start_io()

python/src/deltachat/_build.py

@@ -1,65 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -71,13 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -50,7 +50,7 @@ class Account(object):
             lib.dc_context_unref,
         )
         if self._dc_context == ffi.NULL:
-            raise ValueError("FAILED dc_context_new: {} {}".format(os_name, db_path))
+            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
 
         self._shutdown_event = Event()
         self._event_thread = EventThread(self)
@@ -213,22 +213,40 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, email, name=None):
-        """ create a (new) Contact. If there already is a Contact
-        with that e-mail address, it is unblocked and its name is
-        updated.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        :param email: email-address (text type)
-        :param name: display name for this contact (optional)
+        Calling this method will always resulut in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        `name` is updated if specified.
+
+        :param obj: email-address, Account or Contact instance.
+        :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
+        if isinstance(obj, Account):
+            if not obj.is_configured():
+                raise ValueError("can only add addresses from configured accounts")
+            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
+        elif isinstance(obj, Contact):
+            if obj.account != self:
+                raise ValueError("account mismatch {}".format(obj))
+            addr, displayname = obj.addr, obj.name
+        elif isinstance(obj, str):
+            displayname, addr = parseaddr(obj)
+        else:
+            raise TypeError("don't know how to create chat for %r" % (obj, ))
+
+        if name is None and displayname:
+            name = displayname
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -250,6 +268,13 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -279,53 +304,29 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+    def create_chat(self, obj):
+        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        return self.create_contact(obj).create_chat()
 
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(contact, "id"):
-            if contact.account != self:
-                raise ValueError("Contact belongs to a different Account")
-            contact_id = contact.id
-        else:
-            assert isinstance(contact, int)
-            contact_id = contact
-        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
-        return Chat(self, chat_id)
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
 
-    def create_chat_by_message(self, message):
-        """ create or get an existing chat object for the
-        the specified message.
-
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
-        :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(message, "id"):
-            if message.account != self:
-                raise ValueError("Message belongs to a different Account")
-            msg_id = message.id
-        else:
-            assert isinstance(message, int)
-            msg_id = message
-        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
-        return Chat(self, chat_id)
-
-    def create_group_chat(self, name, verified=False):
+    def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
+        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        return Chat(self, chat_id)
+        chat = Chat(self, chat_id)
+        if contacts is not None:
+            for contact in contacts:
+                chat.add_contact(contact)
+        return chat
 
     def get_chats(self):
         """ return list of chats.
@@ -354,13 +355,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -567,29 +561,24 @@ class Account(object):
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
-        :returns: None (account is configured and with io-scheduling running)
+        :returns: None
         """
         if not self.is_configured():
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
     def configure(self):
+        """ Start configuration process and return a Configtracker instance
+        on which you can block with wait_finish() to get a True/False success
+        value for the configuration process.
+        """
         assert not self.is_configured()
-        assert not hasattr(self, "_configtracker")
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
-        if hasattr(self, "_configtracker"):
-            self.remove_account_plugin(self._configtracker)
-        self._configtracker = ConfigureTracker()
-        self.add_account_plugin(self._configtracker)
+        configtracker = ConfigureTracker(self)
+        self.add_account_plugin(configtracker)
         lib.dc_configure(self._dc_context)
-
-    def wait_configure_finish(self):
-        try:
-            self._configtracker.wait_finish()
-        finally:
-            self.remove_account_plugin(self._configtracker)
-            del self._configtracker
+        return configtracker
 
     def is_started(self):
         return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))

python/src/deltachat/chat.py

@@ -18,6 +18,8 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -328,33 +330,34 @@ class Chat(object):
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, contact):
+    def add_contact(self, obj):
         """ add a contact to this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
+        return contact
 
-    def remove_contact(self, contact):
+    def remove_contact(self, obj):
         """ remove a contact from this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
-        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-
         """
         from .contact import Contact
         dc_array = ffi.gc(
@@ -365,6 +368,14 @@ class Chat(object):
             dc_array, lambda id: Contact(self.account, id))
         )
 
+    def num_contacts(self):
+        """ return number of contacts in this chat. """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return lib.dc_array_get_cnt(dc_array)
+
     def set_profile_image(self, img_path):
         """Set group profile image.
 

python/src/deltachat/const.py

@@ -1,197 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -3,6 +3,8 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
+from .chat import Chat
+from . import const
 
 
 class Contact(object):
@@ -11,6 +13,8 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -36,10 +40,13 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def display_name(self):
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
+    # deprecated alias
+    display_name = name
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -58,6 +65,16 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)
+    def create_chat(self):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        dc_context = self.account._dc_context
+        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
+        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
+        return Chat(self.account, chat_id)
+
+    # deprecated name
+    get_chat = create_chat

python/src/deltachat/direct_imap.py

@@ -0,0 +1,211 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
+import io
+import email
+import ssl
+import pathlib
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
+
+
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+ALL = "1:*"
+
+
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account):
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        ssl_context = ssl.create_default_context()
+
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
+        try:
+            self.conn.logout()
+        except (OSError, IMAPClientError):
+            print("Could not logout direct_imap conn")
+
+    def select_folder(self, foldername):
+        assert not self._idling
+        return self.conn.select_folder(foldername)
+
+    def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
+        if "_" not in config_name:
+            config_name = "configured_{}_folder".format(config_name)
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+
+    def list_folders(self):
+        """ return list of all existing folder names"""
+        assert not self._idling
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
+
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self):
+        assert not self._idling
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
+        assert not self._idling
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.set_flags(messages, [SEEN])
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self):
+        return len(self.get_unread_messages())
+
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[HEADER]']
+                flags = data[FLAGS]
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check()
+        if terminate:
+            self.idle_done()
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/events.py

@@ -28,13 +28,9 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
+    def __init__(self, account):
         self.account = account
-        self.logid = logid
+        self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
@@ -127,6 +123,12 @@ class FFIEventTracker:
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
 
+    def wait_msg_delivered(self, msg):
+        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == msg.chat.id
+        assert ev.data2 == msg.id
+        assert msg.is_out_delivered()
+
 
 class EventThread(threading.Thread):
     """ Event Thread for an account.
@@ -172,7 +174,7 @@ class EventThread(threading.Thread):
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_dc_charpointer(lib.dc_event_get_data3_str(event))
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -88,6 +88,14 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
+    @global_hookspec
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -53,15 +53,19 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
+    def create_chat(self):
+        """ create or get an existing chat (group) object for this message.
+
+        If the message is a deaddrop contact request
+        the sender will become an accepted contact.
+
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
-                lib.dc_msg_unref
-        )
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):

python/src/deltachat/testplugin.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import sys
+import io
 import subprocess
 import queue
 import threading
@@ -16,6 +17,7 @@ from . import Account, const
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
@@ -33,9 +35,6 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
@@ -216,6 +215,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -226,13 +226,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
             ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+                ac.add_account_plugin(FFIEventLogger(ac))
             self._accounts.append(ac)
             return ac
 
@@ -301,31 +303,27 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1.wait_configure_finish()
-            ac1.start_io()
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1.wait_configure_finish()
-            ac1.start_io()
-            ac2.wait_configure_finish()
-            ac2.start_io()
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True, quiet=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
+            self.wait_configure_and_start_io()
             for acc in accounts:
-                acc._configtracker.wait_finish()
-                acc.start_io()
+                acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
@@ -343,14 +341,28 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
                 "-u",
@@ -375,9 +387,42 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    return am
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
@@ -445,4 +490,17 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
+
     return Printer()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # set a report attribute for each phase of a call, which can
+    # be "setup", "call", "teardown"
+
+    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -40,12 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -59,7 +61,10 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
+        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """

python/tests/stress_test_db.py

@@ -17,7 +17,7 @@ def test_db_busy_error(acfactory, tmpdir):
             print("%3.2f %s" % (time.time() - starttime, string))
 
     # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(5, quiet=True)
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
     log("created %s accounts" % len(accounts))
 
     # put a bigfile into each account
@@ -41,7 +41,7 @@ def test_db_busy_error(acfactory, tmpdir):
     # each replier receives all events and sends report events to receive_queue
     repliers = []
     for acc in accounts:
-        replier = AutoReplier(acc, log=log, num_send=1000, num_bigfiles=0, report_func=report_func)
+        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
         acc.add_account_plugin(replier)
         repliers.append(replier)
 
@@ -57,9 +57,9 @@ def test_db_busy_error(acfactory, tmpdir):
             log("timeout waiting for next event")
             pytest.fail("timeout exceeded")
         if report_type == ReportType.exit:
-            replier.log("EXIT".format(alive_count))
+            replier.log("EXIT")
         elif report_type == ReportType.ffi_error:
-            replier.log("ERROR: {}".format(addr, report_args[0]))
+            replier.log("ERROR: {}".format(report_args[0]))
         elif report_type == ReportType.message_echo:
             continue
         else:
@@ -108,14 +108,15 @@ class AutoReplier:
     @deltachat.account_hookimpl
     def ac_incoming_message(self, message):
         if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
             return
-        message.accept_sender_contact()
+        message.create_chat()
         message.mark_seen()
         self.log("incoming message: {}".format(message))
 
         self.current_sent += 1
         # we are still alive, let's send a reply
-        if self.num_bigfiles and self.current_sent % self.num_bigfiles == 0:
+        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
             message.chat.send_text("send big file as reply to: {}".format(message.text))
             msg = message.chat.send_file(self.account.bigfile)
         else:

python/tests/test_increation.py

@@ -36,9 +36,7 @@ def wait_msgs_changed(account, msgs_list):
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -48,9 +46,7 @@ class TestOnlineInCreation:
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -64,9 +60,7 @@ class TestOnlineInCreation:
     def test_forward_increation(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
@@ -80,7 +74,7 @@ class TestOnlineInCreation:
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(c2)
+        chat2.add_contact(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them

python/tests/test_lowlevel.py

@@ -69,8 +69,8 @@ def test_sig():
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
 
-    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    chat = ac1.create_chat_by_contact(contact1)
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]

python/tox.ini

@@ -71,6 +71,8 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
+markers = 
+    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

spec.md

@@ -1,10 +1,12 @@
-# Chat-over-Email specification
+# chat-mail specification
 
-Version 0.30.0
+Version: 0.32.0
+Status:  In-progress 
+Format:  [Semantic Line Breaks](https://sembr.org/)
 
-This document describes how emails can be used
-to implement typical messenger functions
-while staying compatible to existing MUAs.
+This document roughly describes how chat-mail 
+apps use the standard e-mail system 
+to implement typical messenger functions.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -30,17 +32,14 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
-If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD start with the characters `Chat:`
-and SHOULD be an excerpt of the message.
+the `Subject` header SHOULD be `Message from <sender name>`.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -58,7 +57,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Chat: Hello ...
+    Subject: Message from sender@domain
 
     Hello world!
 

src/blob.rs

@@ -8,11 +8,14 @@ use async_std::prelude::*;
 use async_std::{fs, io};
 
 use image::GenericImageView;
+use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::constants::AVATAR_SIZE;
+use crate::config::Config;
+use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
+use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -57,7 +60,7 @@ impl<'a> BlobObject<'a> {
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -370,11 +373,49 @@ impl<'a> BlobObject<'a> {
         let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
 
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
+    }
+
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
+            != Some((Viewtype::Image, "image/jpeg"))
+        {
+            return Ok(());
+        }
+
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
 
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
+
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
+        }
+
+        let img = img.thumbnail(img_wh, img_wh);
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
         Ok(())
     }
 }
@@ -400,7 +441,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {

src/chat.rs

@@ -39,18 +39,9 @@ impl ChatId {
         ChatId(id)
     }
 
-    /// A ChatID which indicates an error.
-    ///
-    /// This is transitional and should not be used in new code.  Do
-    /// not represent errors in a ChatId.
-    pub fn is_error(self) -> bool {
-        self.0 == 0
-    }
-
     /// An unset ChatId
     ///
-    /// Like [ChatId::is_error], from which it is indistinguishable, this is
-    /// transitional and should not be used in new code.
+    /// This is transitional and should not be used in new code.
     pub fn is_unset(self) -> bool {
         self.0 == 0
     }
@@ -431,15 +422,35 @@ impl ChatId {
     }
 
     async fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
-        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
-        self.parent_query(
-            context,
-            "rfc724_mid, mime_in_reply_to, mime_references",
-            collect,
-        )
-        .await
-        .ok()
-        .flatten()
+        let collect =
+            |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?));
+        let (packed, rfc724_mid, mime_in_reply_to, mime_references): (
+            String,
+            String,
+            String,
+            String,
+        ) = self
+            .parent_query(
+                context,
+                "param, rfc724_mid, mime_in_reply_to, mime_references",
+                collect,
+            )
+            .await
+            .ok()
+            .flatten()?;
+
+        let param = packed.parse::<Params>().ok()?;
+        if param.exists(Param::Error) {
+            // Do not reply to error messages.
+            //
+            // An error message could be a group chat message that we failed to decrypt and
+            // assigned to 1:1 chat. A reply to it will show up as a reply to group message
+            // on the other side. To avoid such situations, it is better not to reply to
+            // error messages at all.
+            None
+        } else {
+            Some((rfc724_mid, mime_in_reply_to, mime_references))
+        }
     }
 
     async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
@@ -448,7 +459,7 @@ impl ChatId {
 
         if let Some(ref packed) = packed {
             let param = packed.parse::<Params>()?;
-            Ok(param.exists(Param::GuaranteeE2ee))
+            Ok(!param.exists(Param::Error) && param.exists(Param::GuaranteeE2ee))
         } else {
             // No messages
             Ok(false)
@@ -1341,6 +1352,12 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Er
             .ok_or_else(|| {
                 format_err!("Attachment missing for message of type #{}", msg.viewtype)
             })?;
+
+        if msg.viewtype == Viewtype::Image {
+            if let Err(e) = blob.recode_to_image_size(context).await {
+                warn!(context, "Cannot recode image, using original data: {:?}", e);
+            }
+        }
         msg.param.set(Param::File, blob.as_name());
 
         if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
@@ -1930,20 +1947,19 @@ pub async fn create_group_chat(
         .sql
         .get_rowid(context, "chats", "grpid", grpid)
         .await?;
-    let chat_id = ChatId::new(row_id);
-    if !chat_id.is_error() {
-        if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
-            let mut draft_msg = Message::new(Viewtype::Text);
-            draft_msg.set_text(Some(draft_txt));
-            chat_id.set_draft_raw(context, &mut draft_msg).await;
-        }
 
-        context.emit_event(Event::MsgsChanged {
-            msg_id: MsgId::new(0),
-            chat_id: ChatId::new(0),
-        });
+    let chat_id = ChatId::new(row_id);
+    if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
+        let mut draft_msg = Message::new(Viewtype::Text);
+        draft_msg.set_text(Some(draft_txt));
+        chat_id.set_draft_raw(context, &mut draft_msg).await;
     }
 
+    context.emit_event(Event::MsgsChanged {
+        msg_id: MsgId::new(0),
+        chat_id: ChatId::new(0),
+    });
+
     Ok(chat_id)
 }
 

src/config.rs

@@ -11,7 +11,7 @@ use crate::dc_tools::*;
 use crate::events::Event;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::stock::StockMessage;
+use crate::{scheduler::InterruptInfo, stock::StockMessage};
 
 /// The available configuration keys.
 #[derive(
@@ -104,6 +104,9 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
+    ConfiguredInboxFolder,
+    ConfiguredMvboxFolder,
+    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -137,6 +140,7 @@ impl Context {
         // Default values
         match key {
             Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
@@ -199,17 +203,18 @@ impl Context {
             }
             Config::InboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_inbox().await;
+                self.interrupt_inbox(InterruptInfo::new(false, None)).await;
                 ret
             }
             Config::SentboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_sentbox().await;
+                self.interrupt_sentbox(InterruptInfo::new(false, None))
+                    .await;
                 ret
             }
             Config::MvboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_mvbox().await;
+                self.interrupt_mvbox(InterruptInfo::new(false, None)).await;
                 ret
             }
             Config::Selfstatus => {

src/configure/mod.rs

@@ -4,7 +4,7 @@ mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
 
-use anyhow::{bail, ensure, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use async_std::prelude::*;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
@@ -66,58 +66,11 @@ impl Context {
     }
 
     async fn inner_configure(&self) -> Result<()> {
-        let mut success = false;
-        let mut param_autoconfig: Option<LoginParam> = None;
-
         info!(self, "Configure ...");
 
-        // Variables that are shared between steps:
-        let mut param = LoginParam::from_database(self, "").await;
-        // need all vars here to be mutable because rust thinks the same step could be called multiple times
-        // and also initialize, because otherwise rust thinks it's used while unitilized, even if thats not the case as the loop goes only forward
-        let mut param_domain = "undefined.undefined".to_owned();
-        let mut param_addr_urlencoded: String =
-            "Internal Error: this value should never be used".to_owned();
-        let mut keep_flags = 0;
-
-        let mut step_counter: u8 = 0;
-        let (_s, r) = async_std::sync::channel(1);
-        let mut imap = Imap::new(r);
         let was_configured_before = self.is_configured().await;
-
-        while !self.shall_stop_ongoing().await {
-            step_counter += 1;
-
-            match exec_step(
-                self,
-                &mut imap,
-                &mut param,
-                &mut param_domain,
-                &mut param_autoconfig,
-                &mut param_addr_urlencoded,
-                &mut keep_flags,
-                &mut step_counter,
-            )
-            .await
-            {
-                Ok(step) => {
-                    success = true;
-                    match step {
-                        Step::Continue => {}
-                        Step::Done => break,
-                    }
-                }
-                Err(err) => {
-                    error!(self, "{}", err);
-                    success = false;
-                    break;
-                }
-            }
-        }
-
-        if imap.is_connected() {
-            imap.disconnect(self).await;
-        }
+        let mut param = LoginParam::from_database(self, "").await;
+        let success = configure(self, &mut param).await;
 
         if let Some(provider) = provider::get_provider_info(&param.addr) {
             if !was_configured_before {
@@ -141,329 +94,294 @@ impl Context {
             }
         }
 
-        // remember the entered parameters on success
-        // and restore to last-entered on failure.
-        // this way, the parameters visible to the ui are always in-sync with the current configuration.
-        if success {
-            LoginParam::from_database(self, "")
-                .await
-                .save_to_database(self, "configured_raw_")
-                .await
-                .ok();
+        match success {
+            Ok(_) => {
+                progress!(self, 1000);
+                Ok(())
+            }
+            Err(err) => {
+                error!(self, "Configure Failed: {}", err);
+                progress!(self, 0);
+                Err(err)
+            }
+        }
+    }
+}
 
-            progress!(self, 1000);
-            Ok(())
+async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
+    let mut param_autoconfig: Option<LoginParam> = None;
+    let mut keep_flags = 0;
+
+    // Read login parameters from the database
+    progress!(ctx, 1);
+    ensure!(!param.addr.is_empty(), "Please enter an email address.");
+
+    // Step 1: Load the parameters and check email-address and password
+
+    if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
+        // the used oauth2 addr may differ, check this.
+        // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
+        progress!(ctx, 10);
+        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.mail_pw)
+            .await
+            .and_then(|e| e.parse().ok())
+        {
+            info!(ctx, "Authorized address is {}", oauth2_addr);
+            param.addr = oauth2_addr;
+            ctx.sql
+                .set_raw_config(ctx, "addr", Some(param.addr.as_str()))
+                .await?;
+        }
+        progress!(ctx, 20);
+    }
+    // no oauth? - just continue it's no error
+
+    let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
+    let param_domain = parsed.domain;
+    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
+
+    // Step 2: Autoconfig
+    progress!(ctx, 200);
+
+    // param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then
+    // param.send_pw.is_empty()   -- the password cannot be auto-configured and is no criterion for
+    //                               autoconfig or not
+    if param.mail_server.is_empty()
+        && param.mail_port == 0
+        && param.send_server.is_empty()
+        && param.send_port == 0
+        && param.send_user.is_empty()
+        && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
+    {
+        // no advanced parameters entered by the user: query provider-database or do Autoconfig
+        keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
+        if let Some(new_param) = get_offline_autoconfig(ctx, &param) {
+            // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
+            param_autoconfig = Some(new_param);
+        }
+
+        if param_autoconfig.is_none() {
+            param_autoconfig =
+                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
+        }
+    }
+
+    // C.  Do we have any autoconfig result?
+    progress!(ctx, 500);
+    if let Some(ref cfg) = param_autoconfig {
+        info!(ctx, "Got autoconfig: {}", &cfg);
+        if !cfg.mail_user.is_empty() {
+            param.mail_user = cfg.mail_user.clone();
+        }
+        // all other values are always NULL when entering autoconfig
+        param.mail_server = cfg.mail_server.clone();
+        param.mail_port = cfg.mail_port;
+        param.send_server = cfg.send_server.clone();
+        param.send_port = cfg.send_port;
+        param.send_user = cfg.send_user.clone();
+        param.server_flags = cfg.server_flags;
+        // although param_autoconfig's data are no longer needed from,
+        // it is used to later to prevent trying variations of port/server/logins
+    }
+    param.server_flags |= keep_flags;
+
+    // Step 3: Fill missing fields with defaults
+    if param.mail_server.is_empty() {
+        param.mail_server = format!("imap.{}", param_domain,)
+    }
+    if param.mail_port == 0 {
+        param.mail_port = if 0 != param.server_flags & (0x100 | 0x400) {
+            143
         } else {
-            LoginParam::from_database(self, "configured_raw_")
-                .await
-                .save_to_database(self, "")
-                .await
-                .ok();
-
-            progress!(self, 0);
-            bail!("Configure failed")
+            993
         }
     }
+    if param.mail_user.is_empty() {
+        param.mail_user = param.addr.clone();
+    }
+    if param.send_server.is_empty() && !param.mail_server.is_empty() {
+        param.send_server = param.mail_server.clone();
+        if param.send_server.starts_with("imap.") {
+            param.send_server = param.send_server.replacen("imap", "smtp", 1);
+        }
+    }
+    if param.send_port == 0 {
+        param.send_port = if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
+            587
+        } else if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
+            25
+        } else {
+            465
+        }
+    }
+    if param.send_user.is_empty() && !param.mail_user.is_empty() {
+        param.send_user = param.mail_user.clone();
+    }
+    if param.send_pw.is_empty() && !param.mail_pw.is_empty() {
+        param.send_pw = param.mail_pw.clone()
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
+        param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
+        param.server_flags |= DC_LP_AUTH_NORMAL as i32
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= if param.send_port == 143 {
+            DC_LP_IMAP_SOCKET_STARTTLS as i32
+        } else {
+            DC_LP_IMAP_SOCKET_SSL as i32
+        }
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= if param.send_port == 587 {
+            DC_LP_SMTP_SOCKET_STARTTLS as i32
+        } else if param.send_port == 25 {
+            DC_LP_SMTP_SOCKET_PLAIN as i32
+        } else {
+            DC_LP_SMTP_SOCKET_SSL as i32
+        }
+    }
+
+    // do we have a complete configuration?
+    ensure!(
+        !param.mail_server.is_empty()
+            && param.mail_port != 0
+            && !param.mail_user.is_empty()
+            && !param.mail_pw.is_empty()
+            && !param.send_server.is_empty()
+            && param.send_port != 0
+            && !param.send_user.is_empty()
+            && !param.send_pw.is_empty()
+            && param.server_flags != 0,
+        "Account settings incomplete."
+    );
+
+    progress!(ctx, 600);
+    // try to connect to IMAP - if we did not got an autoconfig,
+    // do some further tries with different settings and username variations
+    let (_s, r) = async_std::sync::channel(1);
+    let mut imap = Imap::new(r);
+
+    try_imap_connections(ctx, param, param_autoconfig.is_some(), &mut imap).await?;
+    progress!(ctx, 800);
+
+    try_smtp_connections(ctx, param, param_autoconfig.is_some()).await?;
+    progress!(ctx, 900);
+
+    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
+        || ctx.get_config_bool(Config::MvboxMove).await;
+
+    imap.configure_folders(ctx, create_mvbox).await?;
+
+    imap.select_with_uidvalidity(ctx, "INBOX")
+        .await
+        .context("could not read INBOX status")?;
+
+    drop(imap);
+
+    progress!(ctx, 910);
+    // configuration success - write back the configured parameters with the
+    // "configured_" prefix; also write the "configured"-flag */
+    // the trailing underscore is correct
+    param.save_to_database(ctx, "configured_").await?;
+    ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
+
+    progress!(ctx, 920);
+
+    e2ee::ensure_secret_key_exists(ctx).await?;
+    info!(ctx, "key generation completed");
+
+    progress!(ctx, 940);
+
+    Ok(())
 }
 
-#[allow(clippy::too_many_arguments)]
-async fn exec_step(
-    ctx: &Context,
-    imap: &mut Imap,
-    param: &mut LoginParam,
-    param_domain: &mut String,
-    param_autoconfig: &mut Option<LoginParam>,
-    param_addr_urlencoded: &mut String,
-    keep_flags: &mut i32,
-    step_counter: &mut u8,
-) -> Result<Step> {
-    const STEP_12_USE_AUTOCONFIG: u8 = 12;
-    const STEP_13_AFTER_AUTOCONFIG: u8 = 13;
+#[derive(Debug, PartialEq, Eq)]
+enum AutoconfigProvider {
+    Mozilla,
+    Outlook,
+}
 
-    match *step_counter {
-        // Read login parameters from the database
-        1 => {
-            progress!(ctx, 1);
-            ensure!(!param.addr.is_empty(), "Please enter an email address.");
-        }
-        // Step 1: Load the parameters and check email-address and password
-        2 => {
-            if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
-                // the used oauth2 addr may differ, check this.
-                // if dc_get_oauth2_addr() is not available in the oauth2 implementation,
-                // just use the given one.
-                progress!(ctx, 10);
-                if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.mail_pw)
-                    .await
-                    .and_then(|e| e.parse().ok())
-                {
-                    info!(ctx, "Authorized address is {}", oauth2_addr);
-                    param.addr = oauth2_addr;
-                    ctx.sql
-                        .set_raw_config(ctx, "addr", Some(param.addr.as_str()))
-                        .await?;
-                }
-                progress!(ctx, 20);
-            }
-            // no oauth? - just continue it's no error
-        }
-        3 => {
-            if let Ok(parsed) = param.addr.parse() {
-                let parsed: EmailAddress = parsed;
-                *param_domain = parsed.domain;
-                *param_addr_urlencoded =
-                    utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
-            } else {
-                bail!("Bad email-address.");
-            }
-        }
-        // Step 2: Autoconfig
-        4 => {
-            progress!(ctx, 200);
+#[derive(Debug, PartialEq, Eq)]
+struct AutoconfigSource {
+    provider: AutoconfigProvider,
+    url: String,
+}
 
-            if param.mail_server.is_empty()
-                && param.mail_port == 0
-            /* && param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then */
-                && param.send_server.is_empty()
-                && param.send_port == 0
-                && param.send_user.is_empty()
-            /* && param.send_pw.is_empty() -- the password cannot be auto-configured and is no criterion for autoconfig or not */
-                && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
-            {
-                // no advanced parameters entered by the user: query provider-database or do Autoconfig
-                *keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
-                if let Some(new_param) = get_offline_autoconfig(ctx, &param) {
-                    // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
-                    *param_autoconfig = Some(new_param);
-                    *step_counter = STEP_12_USE_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
-                }
-            } else {
-                // advanced parameters entered by the user: skip Autoconfig
-                *step_counter = STEP_13_AFTER_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
-            }
-        }
-        /* A.  Search configurations from the domain used in the email-address, prefer encrypted */
-        5 => {
-            if param_autoconfig.is_none() {
-                let url = format!(
+impl AutoconfigSource {
+    fn all(domain: &str, addr: &str) -> [Self; 5] {
+        [
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!(
                     "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    param_domain, param_addr_urlencoded
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        6 => {
-            progress!(ctx, 300);
-            if param_autoconfig.is_none() {
-                // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-                let url = format!(
+                    domain, addr,
+                ),
+            },
+            // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!(
                     "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-                    param_domain, param_addr_urlencoded
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        /* Outlook section start ------------- */
-        /* Outlook uses always SSL but different domains (this comment describes the next two steps) */
-        7 => {
-            progress!(ctx, 310);
-            if param_autoconfig.is_none() {
-                let url = format!("https://{}/autodiscover/autodiscover.xml", param_domain);
-                *param_autoconfig = outlk_autodiscover(ctx, &url, &param).await.ok();
-            }
-        }
-        8 => {
-            progress!(ctx, 320);
-            if param_autoconfig.is_none() {
-                let url = format!(
+                    domain, addr
+                ),
+            },
+            AutoconfigSource {
+                provider: AutoconfigProvider::Outlook,
+                url: format!("https://{}/autodiscover/autodiscover.xml", domain),
+            },
+            // Outlook uses always SSL but different domains (this comment describes the next two steps)
+            AutoconfigSource {
+                provider: AutoconfigProvider::Outlook,
+                url: format!(
                     "https://{}{}/autodiscover/autodiscover.xml",
-                    "autodiscover.", param_domain
-                );
-                *param_autoconfig = outlk_autodiscover(ctx, &url, &param).await.ok();
-            }
-        }
-        /* ----------- Outlook section end */
-        9 => {
-            progress!(ctx, 330);
-            if param_autoconfig.is_none() {
-                let url = format!(
-                    "http://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    param_domain, param_addr_urlencoded
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        10 => {
-            progress!(ctx, 340);
-            if param_autoconfig.is_none() {
-                // do not transfer the email-address unencrypted
-                let url = format!(
-                    "http://{}/.well-known/autoconfig/mail/config-v1.1.xml",
-                    param_domain
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        /* B.  If we have no configuration yet, search configuration in Thunderbird's centeral database */
-        11 => {
-            progress!(ctx, 350);
-            if param_autoconfig.is_none() {
-                /* always SSL for Thunderbird's database */
-                let url = format!("https://autoconfig.thunderbird.net/v1.1/{}", param_domain);
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        /* C.  Do we have any autoconfig result?
-           If you change the match-number here, also update STEP_12_COPY_AUTOCONFIG above
-        */
-        STEP_12_USE_AUTOCONFIG => {
-            progress!(ctx, 500);
-            if let Some(ref cfg) = param_autoconfig {
-                info!(ctx, "Got autoconfig: {}", &cfg);
-                if !cfg.mail_user.is_empty() {
-                    param.mail_user = cfg.mail_user.clone();
-                }
-                param.mail_server = cfg.mail_server.clone(); /* all other values are always NULL when entering autoconfig */
-                param.mail_port = cfg.mail_port;
-                param.send_server = cfg.send_server.clone();
-                param.send_port = cfg.send_port;
-                param.send_user = cfg.send_user.clone();
-                param.server_flags = cfg.server_flags;
-                /* although param_autoconfig's data are no longer needed from,
-                it is used to later to prevent trying variations of port/server/logins */
-            }
-            param.server_flags |= *keep_flags;
-        }
-        // Step 3: Fill missing fields with defaults
-        // If you change the match-number here, also update STEP_13_AFTER_AUTOCONFIG above
-        STEP_13_AFTER_AUTOCONFIG => {
-            if param.mail_server.is_empty() {
-                param.mail_server = format!("imap.{}", param_domain,)
-            }
-            if param.mail_port == 0 {
-                param.mail_port = if 0 != param.server_flags & (0x100 | 0x400) {
-                    143
-                } else {
-                    993
-                }
-            }
-            if param.mail_user.is_empty() {
-                param.mail_user = param.addr.clone();
-            }
-            if param.send_server.is_empty() && !param.mail_server.is_empty() {
-                param.send_server = param.mail_server.clone();
-                if param.send_server.starts_with("imap.") {
-                    param.send_server = param.send_server.replacen("imap", "smtp", 1);
-                }
-            }
-            if param.send_port == 0 {
-                param.send_port = if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
-                    587
-                } else if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
-                    25
-                } else {
-                    465
-                }
-            }
-            if param.send_user.is_empty() && !param.mail_user.is_empty() {
-                param.send_user = param.mail_user.clone();
-            }
-            if param.send_pw.is_empty() && !param.mail_pw.is_empty() {
-                param.send_pw = param.mail_pw.clone()
-            }
-            if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
-                param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-                param.server_flags |= DC_LP_AUTH_NORMAL as i32
-            }
-            if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
-                param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-                param.server_flags |= if param.send_port == 143 {
-                    DC_LP_IMAP_SOCKET_STARTTLS as i32
-                } else {
-                    DC_LP_IMAP_SOCKET_SSL as i32
-                }
-            }
-            if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
-                param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-                param.server_flags |= if param.send_port == 587 {
-                    DC_LP_SMTP_SOCKET_STARTTLS as i32
-                } else if param.send_port == 25 {
-                    DC_LP_SMTP_SOCKET_PLAIN as i32
-                } else {
-                    DC_LP_SMTP_SOCKET_SSL as i32
-                }
-            }
-            /* do we have a complete configuration? */
-            if param.mail_server.is_empty()
-                || param.mail_port == 0
-                || param.mail_user.is_empty()
-                || param.mail_pw.is_empty()
-                || param.send_server.is_empty()
-                || param.send_port == 0
-                || param.send_user.is_empty()
-                || param.send_pw.is_empty()
-                || param.server_flags == 0
-            {
-                bail!("Account settings incomplete.");
-            }
-        }
-        14 => {
-            progress!(ctx, 600);
-            /* try to connect to IMAP - if we did not got an autoconfig,
-            do some further tries with different settings and username variations */
-            try_imap_connections(ctx, param, param_autoconfig.is_some(), imap).await?;
-        }
-        15 => {
-            progress!(ctx, 800);
-            try_smtp_connections(ctx, param, param_autoconfig.is_some()).await?;
-        }
-        16 => {
-            progress!(ctx, 900);
+                    "autodiscover.", domain
+                ),
+            },
+            // always SSL for Thunderbird's database
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!("https://autoconfig.thunderbird.net/v1.1/{}", domain),
+            },
+        ]
+    }
 
-            let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
-                || ctx.get_config_bool(Config::MvboxMove).await;
+    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<LoginParam> {
+        let params = match self.provider {
+            AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
+            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url, &param).await?,
+        };
 
-            if let Err(err) = imap.configure_folders(ctx, create_mvbox).await {
-                bail!("configuring folders failed: {:?}", err);
-            }
+        Ok(params)
+    }
+}
 
-            if let Err(err) = imap.select_with_uidvalidity(ctx, "INBOX").await {
-                bail!("could not read INBOX status: {:?}", err);
-            }
-        }
-        17 => {
-            progress!(ctx, 910);
-            // configuration success - write back the configured parameters with the
-            // "configured_" prefix; also write the "configured"-flag */
-            // the trailing underscore is correct
-            param.save_to_database(ctx, "configured_").await?;
-            ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
-        }
-        18 => {
-            progress!(ctx, 920);
-            // we generate the keypair just now - we could also postpone this until the first message is sent, however,
-            // this may result in a unexpected and annoying delay when the user sends his very first message
-            // (~30 seconds on a Moto G4 play) and might looks as if message sending is always that slow.
-            e2ee::ensure_secret_key_exists(ctx).await?;
-            info!(ctx, "key generation completed");
-            progress!(ctx, 940);
-            return Ok(Step::Done);
-        }
-        _ => {
-            bail!("Internal error: step counter out of bound");
+/// Retrieve available autoconfigurations.
+///
+/// A Search configurations from the domain used in the email-address, prefer encrypted
+/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
+async fn get_autoconfig(
+    ctx: &Context,
+    param: &LoginParam,
+    param_domain: &str,
+    param_addr_urlencoded: &str,
+) -> Option<LoginParam> {
+    let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
+
+    let mut progress = 300;
+    for source in &sources {
+        let res = source.fetch(ctx, param).await;
+        progress!(ctx, progress);
+        progress += 10;
+        if let Ok(res) = res {
+            return Some(res);
         }
     }
 
-    Ok(Step::Continue)
+    None
 }
 
-#[derive(Debug)]
-enum Step {
-    Done,
-    Continue,
-}
-
-#[allow(clippy::unnecessary_unwrap)]
 fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
     info!(
         context,
@@ -479,37 +397,35 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
                 // however, rewriting the code to "if let" would make things less obvious,
                 // esp. if we allow more combinations of servers (pop, jmap).
                 // therefore, #[allow(clippy::unnecessary_unwrap)] is added above.
-                if imap.is_some() && smtp.is_some() {
-                    let imap = imap.unwrap();
-                    let smtp = smtp.unwrap();
+                if let Some(imap) = imap {
+                    if let Some(smtp) = smtp {
+                        let mut p = LoginParam::new();
+                        p.addr = param.addr.clone();
 
-                    let mut p = LoginParam::new();
-                    p.addr = param.addr.clone();
+                        p.mail_server = imap.hostname.to_string();
+                        p.mail_user = imap.apply_username_pattern(param.addr.clone());
+                        p.mail_port = imap.port as i32;
+                        p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.server_flags |= match imap.socket {
+                            provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
+                            provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
+                        };
 
-                    p.mail_server = imap.hostname.to_string();
-                    p.mail_user = imap.apply_username_pattern(param.addr.clone());
-                    p.mail_port = imap.port as i32;
-                    p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                    p.server_flags |= match imap.socket {
-                        provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
-                        provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
-                    };
+                        p.send_server = smtp.hostname.to_string();
+                        p.send_user = smtp.apply_username_pattern(param.addr.clone());
+                        p.send_port = smtp.port as i32;
+                        p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.server_flags |= match smtp.socket {
+                            provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
+                            provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
+                        };
 
-                    p.send_server = smtp.hostname.to_string();
-                    p.send_user = smtp.apply_username_pattern(param.addr.clone());
-                    p.send_port = smtp.port as i32;
-                    p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                    p.server_flags |= match smtp.socket {
-                        provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
-                        provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
-                    };
-
-                    info!(context, "offline autoconfig found: {}", p);
-                    return Some(p);
-                } else {
-                    info!(context, "offline autoconfig found, but no servers defined");
-                    return None;
+                        info!(context, "offline autoconfig found: {}", p);
+                        return Some(p);
+                    }
                 }
+                info!(context, "offline autoconfig found, but no servers defined");
+                return None;
             }
             provider::Status::BROKEN => {
                 info!(context, "offline autoconfig found, provider is broken");
@@ -523,19 +439,32 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
 
 async fn try_imap_connections(
     context: &Context,
-    mut param: &mut LoginParam,
+    param: &mut LoginParam,
     was_autoconfig: bool,
     imap: &mut Imap,
-) -> Result<bool> {
-    // progress 650 and 660
-    if let Ok(val) = try_imap_connection(context, &mut param, was_autoconfig, 0, imap).await {
-        return Ok(val);
-    }
-    progress!(context, 670);
-    param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-    param.server_flags |= DC_LP_IMAP_SOCKET_SSL;
-    param.mail_port = 993;
+) -> Result<()> {
+    // manually_set_param is used to check whether a particular setting was set manually by the user.
+    // If yes, we do not want to change it to avoid confusing error messages
+    // (you set port 443, but the app tells you it couldn't connect on port 993).
+    let manually_set_param = LoginParam::from_database(context, "").await;
 
+    // progress 650 and 660
+    if try_imap_connection(context, param, &manually_set_param, was_autoconfig, 0, imap)
+        .await
+        .is_ok()
+    {
+        return Ok(()); // we directly return here if it was autoconfig or the connection succeeded
+    }
+
+    progress!(context, 670);
+    // try_imap_connection() changed the flags and port. Change them back:
+    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
+        param.server_flags |= DC_LP_IMAP_SOCKET_SSL;
+    }
+    if manually_set_param.mail_port == 0 {
+        param.mail_port = 993;
+    }
     if let Some(at) = param.mail_user.find('@') {
         param.mail_user = param.mail_user.split_at(at).0.to_string();
     }
@@ -543,35 +472,43 @@ async fn try_imap_connections(
         param.send_user = param.send_user.split_at(at).0.to_string();
     }
     // progress 680 and 690
-    try_imap_connection(context, &mut param, was_autoconfig, 1, imap).await
+    try_imap_connection(context, param, &manually_set_param, was_autoconfig, 1, imap).await
 }
 
 async fn try_imap_connection(
     context: &Context,
     param: &mut LoginParam,
+    manually_set_param: &LoginParam,
     was_autoconfig: bool,
     variation: usize,
     imap: &mut Imap,
-) -> Result<bool> {
-    if try_imap_one_param(context, &param, imap).await.is_ok() {
-        return Ok(true);
+) -> Result<()> {
+    if try_imap_one_param(context, param, imap).await.is_ok() {
+        return Ok(());
     }
     if was_autoconfig {
-        return Ok(false);
+        return Ok(());
     }
+
     progress!(context, 650 + variation * 30);
-    param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-    param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS;
-    if try_imap_one_param(context, &param, imap).await.is_ok() {
-        return Ok(true);
+
+    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
+        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS;
+
+        if try_imap_one_param(context, &param, imap).await.is_ok() {
+            return Ok(());
+        }
     }
 
     progress!(context, 660 + variation * 30);
-    param.mail_port = 143;
 
-    try_imap_one_param(context, &param, imap).await?;
-
-    Ok(true)
+    if manually_set_param.mail_port == 0 {
+        param.mail_port = 143;
+        try_imap_one_param(context, param, imap).await
+    } else {
+        Err(format_err!("no more possible configs"))
+    }
 }
 
 async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Imap) -> Result<()> {
@@ -590,41 +527,51 @@ async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Im
         return Ok(());
     }
 
-    if context.shall_stop_ongoing().await {
-        bail!("Interrupted");
-    }
-
     bail!("Could not connect: {}", inf);
 }
 
 async fn try_smtp_connections(
     context: &Context,
-    mut param: &mut LoginParam,
+    param: &mut LoginParam,
     was_autoconfig: bool,
 ) -> Result<()> {
+    // manually_set_param is used to check whether a particular setting was set manually by the user.
+    // If yes, we do not want to change it to avoid confusing error messages
+    // (you set port 443, but the app tells you it couldn't connect on port 993).
+    let manually_set_param = LoginParam::from_database(context, "").await;
+
     let mut smtp = Smtp::new();
-    /* try to connect to SMTP - if we did not got an autoconfig, the first try was SSL-465 and we do a second try with STARTTLS-587 */
-    if try_smtp_one_param(context, &param, &mut smtp).await.is_ok() {
+    // try to connect to SMTP - if we did not got an autoconfig, the first try was SSL-465 and we do
+    // a second try with STARTTLS-587
+    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
         return Ok(());
     }
     if was_autoconfig {
         return Ok(());
     }
     progress!(context, 850);
-    param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-    param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    param.send_port = 587;
 
-    if try_smtp_one_param(context, &param, &mut smtp).await.is_ok() {
+    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
+    }
+    if manually_set_param.send_port == 0 {
+        param.send_port = 587;
+    }
+
+    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
         return Ok(());
     }
     progress!(context, 860);
-    param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-    param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    param.send_port = 25;
-    try_smtp_one_param(context, &param, &mut smtp).await?;
 
-    Ok(())
+    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
+    }
+    if manually_set_param.send_port == 0 {
+        param.send_port = 25;
+    }
+    try_smtp_one_param(context, param, &mut smtp).await
 }
 
 async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Smtp) -> Result<()> {

src/constants.rs

@@ -227,6 +227,10 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
+// max. width/height of images
+pub const BALANCED_IMAGE_SIZE: u32 = 1280;
+pub const WORSE_IMAGE_SIZE: u32 = 640;
+
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 

src/contact.rs

@@ -3,6 +3,8 @@
 use async_std::path::PathBuf;
 use deltachat_derive::*;
 use itertools::Itertools;
+use lazy_static::lazy_static;
+use regex::Regex;
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
@@ -12,7 +14,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
 use crate::mimeparser::AvatarAction;
@@ -238,6 +240,8 @@ impl Contact {
             "Cannot create contact with empty address"
         );
 
+        let (name, addr) = sanitize_name_and_addr(name, addr);
+
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
@@ -512,8 +516,9 @@ impl Contact {
         let mut modify_cnt = 0;
 
         for (name, addr) in split_address_book(addr_book.as_ref()).into_iter() {
+            let (name, addr) = sanitize_name_and_addr(name, addr);
             let name = normalize_name(name);
-            match Contact::add_or_lookup(context, name, addr, Origin::AddressBook).await {
+            match Contact::add_or_lookup(context, name, &addr, Origin::AddressBook).await {
                 Err(err) => {
                     warn!(
                         context,
@@ -691,18 +696,20 @@ impl Contact {
                     })
                     .await;
                 ret += &p;
-                let self_key = Key::from(SignedPublicKey::load_self(context).await?);
                 let p = context.stock_str(StockMessage::FingerPrints).await;
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key.formatted_fingerprint();
+                let fingerprint_self = SignedPublicKey::load_self(context)
+                    .await?
+                    .fingerprint()
+                    .to_string();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
-                    .map(|k| k.formatted_fingerprint())
+                    .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 let fingerprint_other_unverified = peerstate
                     .peek_key(PeerstateVerifiedStatus::Unverified)
-                    .map(|k| k.formatted_fingerprint())
+                    .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 if loginparam.addr < peerstate.addr {
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
@@ -1028,6 +1035,24 @@ pub fn addr_normalize(addr: &str) -> &str {
     norm
 }
 
+fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
+    lazy_static! {
+        static ref ADDR_WITH_NAME_REGEX: Regex = Regex::new("(.*)<(.*)>").unwrap();
+    }
+    if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
+        (
+            if name.as_ref().is_empty() {
+                normalize_name(&captures[1])
+            } else {
+                name.as_ref().to_string()
+            },
+            captures[2].to_string(),
+        )
+    } else {
+        (name.as_ref().to_string(), addr.as_ref().to_string())
+    }
+}
+
 async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
     if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return;
@@ -1202,6 +1227,10 @@ mod tests {
         assert_eq!(may_be_valid_addr("u@d.tt"), true);
         assert_eq!(may_be_valid_addr("u@.tt"), false);
         assert_eq!(may_be_valid_addr("@d.tt"), false);
+        assert_eq!(may_be_valid_addr("<da@d.tt"), false);
+        assert_eq!(may_be_valid_addr("sk <@d.tt>"), false);
+        assert_eq!(may_be_valid_addr("as@sd.de>"), false);
+        assert_eq!(may_be_valid_addr("ask dkl@dd.tt"), false);
     }
 
     #[test]
@@ -1284,9 +1313,10 @@ mod tests {
             "Name two\ntwo@deux.net\n",
             "Invalid\n+1234567890\n", // invalid, should be ignored
             "\nthree@drei.sam\n",
-            "Name two\ntwo@deux.net\n" // should not be added again
+            "Name two\ntwo@deux.net\n", // should not be added again
+            "\nWonderland, Alice <alice@w.de>\n",
         );
-        assert_eq!(Contact::add_address_book(&t.ctx, book).await.unwrap(), 3);
+        assert_eq!(Contact::add_address_book(&t.ctx, book).await.unwrap(), 4);
 
         // check first added contact, this does not modify because of lower origin
         let (contact_id, sth_modified) =
@@ -1362,6 +1392,19 @@ mod tests {
         assert_eq!(contact.get_name_n_addr(), "schnucki (three@drei.sam)");
         assert!(!contact.is_blocked());
 
+        // Fourth contact:
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "", "alice@w.de", Origin::IncomingUnknownTo)
+                .await
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::None);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "Alice Wonderland");
+        assert_eq!(contact.get_display_name(), "Alice Wonderland");
+        assert_eq!(contact.get_addr(), "alice@w.de");
+        assert_eq!(contact.get_name_n_addr(), "Alice Wonderland (alice@w.de)");
+
         // check SELF
         let contact = Contact::load_from_db(&t.ctx, DC_CONTACT_ID_SELF)
             .await
@@ -1528,4 +1571,50 @@ mod tests {
         assert!(addr_cmp(" aa@aa.ORG ", "AA@AA.ORG"));
         assert!(addr_cmp(" mailto:AA@AA.ORG", "Aa@Aa.orG"));
     }
+
+    #[async_std::test]
+    async fn test_name_in_address() {
+        let t = dummy_context().await;
+
+        let contact_id = Contact::create(&t.ctx, "", "<dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        let contact_id = Contact::create(&t.ctx, "", "Mueller, Dave <dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "Dave Mueller");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        let contact_id = Contact::create(&t.ctx, "name1", "name2 <dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "name1");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        assert!(Contact::create(&t.ctx, "", "<dskjfdslk@sadklj.dk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "<dskjf>dslk@sadklj.dk>")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjfdslksadklj.dk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjfdslk@sadklj.dk>")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjf@dslk@sadkljdk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjf dslk@d.e").await.is_err());
+        assert!(Contact::create(&t.ctx, "", "<dskjf dslk@sadklj.dk")
+            .await
+            .is_err());
+    }
 }

src/context.rs

@@ -15,7 +15,7 @@ use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::{Event, EventEmitter, Events};
 use crate::job::{self, Action};
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
 use crate::message::{self, Message, MessengerMessage, MsgId};
@@ -138,10 +138,15 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        assert!(!self.is_io_running().await, "context is already running");
+        if self.is_io_running().await {
+            info!(self, "IO is already running");
+            return;
+        }
 
-        let l = &mut *self.inner.scheduler.write().await;
-        l.start(self.clone()).await;
+        {
+            let l = &mut *self.inner.scheduler.write().await;
+            l.start(self.clone()).await;
+        }
     }
 
     /// Returns if the IO scheduler is running.
@@ -152,6 +157,11 @@ impl Context {
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
+        if !self.is_io_running().await {
+            info!(self, "IO is not running");
+            return;
+        }
+
         self.inner.stop_io().await;
     }
 
@@ -275,7 +285,7 @@ impl Context {
             .query_get_value(self, "SELECT COUNT(*) FROM acpeerstates;", paramsv![])
             .await;
         let fingerprint_str = match SignedPublicKey::load_self(self).await {
-            Ok(key) => Key::from(key).fingerprint(),
+            Ok(key) => key.fingerprint().hex(),
             Err(err) => format!("<key failure: {}>", err),
         };
 
@@ -290,13 +300,11 @@ impl Context {
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
+            .get_config(Config::ConfiguredSentboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
+            .get_config(Config::ConfiguredMvboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
 
@@ -432,33 +440,19 @@ impl Context {
             .unwrap_or_default()
     }
 
-    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        folder_name.as_ref() == "INBOX"
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
-            .await;
-        if let Some(name) = sentbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
-            .await;
-
-        if let Some(name) = mvbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {

src/dc_receive_imf.rs

@@ -10,7 +10,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, Result};
+use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
@@ -313,8 +313,6 @@ async fn add_parts(
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
-    let mut sort_timestamp = 0;
-    let mut rcvd_timestamp = 0;
     let mut mime_in_reply_to = String::new();
     let mut mime_references = String::new();
     let mut incoming_origin = incoming_origin;
@@ -601,17 +599,9 @@ async fn add_parts(
     }
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
-    calc_timestamps(
-        context,
-        *chat_id,
-        from_id,
-        *sent_timestamp,
-        !seen,
-        &mut sort_timestamp,
-        sent_timestamp,
-        &mut rcvd_timestamp,
-    )
-    .await;
+    let rcvd_timestamp = time();
+    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, !seen).await;
+    *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
     chat_id.unarchive(context).await?;
@@ -749,6 +739,31 @@ async fn add_parts(
         }
     }
 
+    async fn update_last_subject(
+        context: &Context,
+        chat_id: ChatId,
+        mime_parser: &MimeMessage,
+    ) -> Result<()> {
+        let mut chat = Chat::load_from_db(context, chat_id).await?;
+        chat.param.set(
+            Param::LastSubject,
+            mime_parser
+                .get_subject()
+                .ok_or_else(|| format_err!("No subject in email"))?,
+        );
+        chat.update_param(context).await?;
+        Ok(())
+    }
+    update_last_subject(context, chat_id, mime_parser)
+        .await
+        .unwrap_or_else(|e| {
+            warn!(
+                context,
+                "Could not update LastSubject of chat: {}",
+                e.to_string()
+            )
+        });
+
     Ok(())
 }
 
@@ -812,41 +827,38 @@ async fn save_locations(
     }
 }
 
-#[allow(clippy::too_many_arguments)]
-async fn calc_timestamps(
+async fn calc_sort_timestamp(
     context: &Context,
-    chat_id: ChatId,
-    from_id: u32,
     message_timestamp: i64,
+    chat_id: ChatId,
     is_fresh_msg: bool,
-    sort_timestamp: &mut i64,
-    sent_timestamp: &mut i64,
-    rcvd_timestamp: &mut i64,
-) {
-    *rcvd_timestamp = time();
-    *sent_timestamp = message_timestamp;
-    if *sent_timestamp > *rcvd_timestamp {
-        *sent_timestamp = *rcvd_timestamp
-    }
-    *sort_timestamp = message_timestamp;
+) -> i64 {
+    let mut sort_timestamp = message_timestamp;
+
+    // get newest non fresh message for this chat
+    // update sort_timestamp if less than that
     if is_fresh_msg {
         let last_msg_time: Option<i64> = context
             .sql
             .query_get_value(
                 context,
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? and from_id!=? AND timestamp>=?",
-                paramsv![chat_id, from_id as i32, *sort_timestamp],
+                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
+                paramsv![chat_id, MessageState::InFresh],
             )
             .await;
+
         if let Some(last_msg_time) = last_msg_time {
-            if last_msg_time > 0 && *sort_timestamp <= last_msg_time {
-                *sort_timestamp = last_msg_time + 1;
+            if last_msg_time > sort_timestamp {
+                sort_timestamp = last_msg_time;
             }
         }
     }
-    if *sort_timestamp >= dc_smeared_time(context).await {
-        *sort_timestamp = dc_create_smeared_timestamp(context).await;
+
+    if sort_timestamp >= dc_smeared_time(context).await {
+        sort_timestamp = dc_create_smeared_timestamp(context).await;
     }
+
+    sort_timestamp
 }
 
 /// This function tries extracts the group-id from the message and returns the
@@ -889,32 +901,29 @@ async fn create_or_lookup_group(
     }
 
     if grpid.is_empty() {
-        if let Some(value) = mime_parser.get(HeaderDef::MessageId) {
-            if let Some(extracted_grpid) = dc_extract_grpid_from_rfc724_mid(&value) {
-                grpid = extracted_grpid.to_string();
-            }
-        }
-        if grpid.is_empty() {
-            if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
-                grpid = extracted_grpid.to_string();
-            } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References)
-            {
-                grpid = extracted_grpid.to_string();
-            } else {
-                return create_or_lookup_adhoc_group(
-                    context,
-                    mime_parser,
-                    allow_creation,
-                    create_blocked,
-                    from_id,
-                    to_ids,
-                )
-                .await
-                .map_err(|err| {
-                    info!(context, "could not create adhoc-group: {:?}", err);
-                    err
-                });
-            }
+        if let Some(extracted_grpid) = mime_parser
+            .get(HeaderDef::MessageId)
+            .and_then(|value| dc_extract_grpid_from_rfc724_mid(&value))
+        {
+            grpid = extracted_grpid.to_string();
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
+            grpid = extracted_grpid.to_string();
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) {
+            grpid = extracted_grpid.to_string();
+        } else {
+            return create_or_lookup_adhoc_group(
+                context,
+                mime_parser,
+                allow_creation,
+                create_blocked,
+                from_id,
+                to_ids,
+            )
+            .await
+            .map_err(|err| {
+                info!(context, "could not create adhoc-group: {:?}", err);
+                err
+            });
         }
     }
     // now we have a grpid that is non-empty
@@ -1001,7 +1010,7 @@ async fn create_or_lookup_group(
     let (mut chat_id, chat_id_verified, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
         .await
         .unwrap_or((ChatId::new(0), false, Blocked::Not));
-    if !chat_id.is_error() {
+    if !chat_id.is_unset() {
         if chat_id_verified {
             if let Err(err) =
                 check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
@@ -1032,7 +1041,7 @@ async fn create_or_lookup_group(
         .await
         .unwrap_or_default();
 
-    if chat_id.is_error()
+    if chat_id.is_unset()
             && !mime_parser.is_mailinglist_message()
             && !grpid.is_empty()
             && grpname.is_some()
@@ -1205,10 +1214,6 @@ async fn create_or_lookup_adhoc_group(
     from_id: u32,
     to_ids: &ContactIds,
 ) -> Result<(ChatId, Blocked)> {
-    // if we're here, no grpid was found, check if there is an existing
-    // ad-hoc group matching the to-list or if we should and can create one
-    // (we do not want to heuristically look at the likely mangled Subject)
-
     if mime_parser.is_mailinglist_message() {
         // XXX we could parse List-* headers and actually create and
         // manage a mailing list group, eventually
@@ -1219,6 +1224,10 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
+    // if we're here, no grpid was found, check if there is an existing
+    // ad-hoc group matching the to-list or if we should and can create one
+    // (we do not want to heuristically look at the likely mangled Subject)
+
     let mut member_ids: Vec<u32> = to_ids.iter().copied().collect();
     if !member_ids.contains(&from_id) {
         member_ids.push(from_id);
@@ -1271,6 +1280,24 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
+    if mime_parser.decrypting_failed {
+        // Do not create a new ad-hoc group if the message cannot be
+        // decrypted.
+        //
+        // The subject may be encrypted and contain a placeholder such
+        // as "...". Besides that, it is possible that the message was
+        // sent to a valid, yet unknown group, which was rejected
+        // because Chat-Group-Name, which is in the encrypted part,
+        // was not found. Generating a new ID in this case would
+        // result in creation of a twin group with a different group
+        // ID.
+        warn!(
+            context,
+            "not creating ad-hoc group for message that cannot be decrypted"
+        );
+        return Ok((ChatId::new(0), Blocked::Not));
+    }
+
     // we do not check if the message is a reply to another group, this may result in
     // chats with unclear member list. instead we create a new group in the following lines ...
 
@@ -1740,7 +1767,7 @@ mod tests {
     use crate::chat::ChatVisibility;
     use crate::chatlist::Chatlist;
     use crate::message::Message;
-    use crate::test_utils::{dummy_context, TestContext};
+    use crate::test_utils::{configured_offline_context, dummy_context};
 
     #[test]
     fn test_hex_hash() {
@@ -1836,23 +1863,6 @@ mod tests {
         assert!(!is_msgrmsg_rfc724_mid(&t.ctx, "nonexistant@message.id").await);
     }
 
-    async fn configured_offline_context() -> TestContext {
-        let t = dummy_context().await;
-        t.ctx
-            .set_config(Config::Addr, Some("alice@example.org"))
-            .await
-            .unwrap();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.org"))
-            .await
-            .unwrap();
-        t.ctx
-            .set_config(Config::Configured, Some("1"))
-            .await
-            .unwrap();
-        t
-    }
-
     static MSGRMSG: &[u8] = b"From: Bob <bob@example.org>\n\
                     To: alice@example.org\n\
                     Chat-Version: 1.0\n\

src/dc_tools.rs

@@ -482,15 +482,6 @@ pub struct InvalidEmailError {
     addr: String,
 }
 
-impl InvalidEmailError {
-    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
-        InvalidEmailError {
-            message: msg.into(),
-            addr: addr.into(),
-        }
-    }
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -530,17 +521,24 @@ impl FromStr for EmailAddress {
 
     /// Performs a dead-simple parse of an email address.
     fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        if input.is_empty() {
-            return Err(InvalidEmailError::new("empty string is not valid", input));
-        }
-        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
-
         let err = |msg: &str| {
             Err(InvalidEmailError {
                 message: msg.to_string(),
                 addr: input.to_string(),
             })
         };
+        if input.is_empty() {
+            return err("empty string is not valid");
+        }
+        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
+
+        if input
+            .chars()
+            .any(|c| c.is_whitespace() || c == '<' || c == '>')
+        {
+            return err("Email must not contain whitespaces, '>' or '<'");
+        }
+
         match &parts[..] {
             [domain, local] => {
                 if local.is_empty() {

src/e2ee.rs

@@ -11,7 +11,7 @@ use crate::context::Context;
 use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -87,30 +87,29 @@ impl EncryptHelper {
 
     /// Tries to encrypt the passed in `mail`.
     pub async fn encrypt(
-        &mut self,
+        self,
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: &[(Option<Peerstate<'_>>, &str)],
+        peerstates: Vec<(Option<Peerstate<'_>>, &str)>,
     ) -> Result<String> {
-        let mut keyring = Keyring::default();
+        let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
         for (peerstate, addr) in peerstates
-            .iter()
-            .filter_map(|(state, addr)| state.as_ref().map(|s| (s, addr)))
+            .into_iter()
+            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate.peek_key(min_verified).ok_or_else(|| {
+            let key = peerstate.take_key(min_verified).ok_or_else(|| {
                 format_err!("proper enc-key for {} missing, cannot encrypt", addr)
             })?;
-            keyring.add_ref(key);
+            keyring.add(key);
         }
-        let public_key = Key::from(self.public_key.clone());
-        keyring.add_ref(&public_key);
-        let sign_key = Key::from(SignedSecretKey::load_self(context).await?);
+        keyring.add(self.public_key.clone());
+        let sign_key = SignedSecretKey::load_self(context).await?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
-        let ctext = pgp::pk_encrypt(&raw_message, &keyring, Some(&sign_key))?;
+        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key)).await?;
 
         Ok(ctext)
     }
@@ -120,7 +119,7 @@ pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
@@ -151,41 +150,33 @@ pub async fn try_decrypt(
     }
 
     /* possibly perform decryption */
-    let mut private_keyring = Keyring::default();
-    let mut public_keyring_for_validate = Keyring::default();
-    let mut out_mail = None;
+    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
+    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
-    let self_addr = context.get_config(Config::ConfiguredAddr).await;
 
-    if let Some(self_addr) = self_addr {
-        if private_keyring
-            .load_self_private_for_decrypting(context, self_addr, &context.sql)
-            .await
-        {
-            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-                peerstate = Peerstate::from_addr(&context, &from).await;
-            }
-            if let Some(ref peerstate) = peerstate {
-                if peerstate.degrade_event.is_some() {
-                    handle_degrade_event(context, &peerstate).await?;
-                }
-                if let Some(ref key) = peerstate.gossip_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-                if let Some(ref key) = peerstate.public_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-            }
-
-            out_mail = decrypt_if_autocrypt_message(
-                context,
-                mail,
-                &private_keyring,
-                &public_keyring_for_validate,
-                &mut signatures,
-            )?;
+    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
+        peerstate = Peerstate::from_addr(&context, &from).await;
+    }
+    if let Some(peerstate) = peerstate {
+        if peerstate.degrade_event.is_some() {
+            handle_degrade_event(context, &peerstate).await?;
+        }
+        if let Some(key) = peerstate.gossip_key {
+            public_keyring_for_validate.add(key);
+        }
+        if let Some(key) = peerstate.public_key {
+            public_keyring_for_validate.add(key);
         }
     }
+
+    let out_mail = decrypt_if_autocrypt_message(
+        context,
+        mail,
+        private_keyring,
+        public_keyring_for_validate,
+        &mut signatures,
+    )
+    .await?;
     Ok((out_mail, signatures))
 }
 
@@ -216,12 +207,12 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
     Ok(&mail.subparts[1])
 }
 
-fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
     mail: &ParsedMail<'a>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     //  The returned bool is true if we detected an Autocrypt-encrypted
     // message and successfully decrypted it. Decryption then modifies the
@@ -240,21 +231,20 @@ fn decrypt_if_autocrypt_message<'a>(
     info!(context, "Detected Autocrypt-mime message");
 
     decrypt_part(
-        context,
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
         ret_valid_signatures,
     )
+    .await
 }
 
 /// Returns Ok(None) if nothing encrypted was found.
-fn decrypt_part(
-    _context: &Context,
+async fn decrypt_part(
     mail: &ParsedMail<'_>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
@@ -263,11 +253,12 @@ fn decrypt_part(
         ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
         let plain = pgp::pk_decrypt(
-            &data,
-            &private_keyring,
-            &public_keyring_for_validate,
+            data,
+            private_keyring,
+            public_keyring_for_validate,
             Some(ret_valid_signatures),
-        )?;
+        )
+        .await?;
 
         ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
         return Ok(Some(plain));

src/headerdef.rs

@@ -34,6 +34,7 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
+    ChatUploadUrl,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,

src/imap/client.rs

@@ -7,7 +7,7 @@ use async_imap::{
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
+use crate::login_param::dc_build_tls;
 
 use super::session::SessionStream;
 
@@ -78,10 +78,10 @@ impl Client {
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        certificate_checks: CertificateChecks,
+        strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
+        let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
@@ -118,16 +118,12 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
+    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {
             let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(certificate_checks);
+            let tls = dc_build_tls(strict_tls);
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();

src/imap/idle.rs

@@ -4,7 +4,7 @@ use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::context::Context;
+use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::select_folder;
 use super::session::Session;
@@ -34,7 +34,11 @@ impl Imap {
         self.config.can_idle
     }
 
-    pub async fn idle(&mut self, context: &Context, watch_folder: Option<String>) -> Result<()> {
+    pub async fn idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> Result<InterruptInfo> {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
@@ -46,6 +50,7 @@ impl Imap {
 
         let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
+        let mut info = Default::default();
 
         if let Some(session) = session {
             let mut handle = session.idle();
@@ -55,6 +60,11 @@ impl Imap {
 
             let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
 
+            enum Event {
+                IdleResponse(IdleResponse),
+                Interrupt(InterruptInfo),
+            }
+
             if self.skip_next_idle_wait {
                 // interrupt_idle has happened before we
                 // provided self.interrupt
@@ -65,23 +75,27 @@ impl Imap {
                 info!(context, "Idle wait was skipped");
             } else {
                 info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.race(
-                    self.idle_interrupt
-                        .recv()
-                        .map(|_| Ok(IdleResponse::ManualInterrupt)),
+                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
+                    self.idle_interrupt.recv().map(|probe_network| {
+                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+                    }),
                 );
 
                 match fut.await {
-                    Ok(IdleResponse::NewData(_)) => {
+                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
                         info!(context, "Idle has NewData");
                     }
                     // TODO: idle_wait does not distinguish manual interrupts
                     // from Timeouts if we would know it's a Timeout we could bail
                     // directly and reconnect .
-                    Ok(IdleResponse::Timeout) => {
+                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
                         info!(context, "Idle-wait timeout or interruption");
                     }
-                    Ok(IdleResponse::ManualInterrupt) => {
+                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                        info!(context, "Idle wait was interrupted");
+                    }
+                    Ok(Event::Interrupt(i)) => {
+                        info = i;
                         info!(context, "Idle wait was interrupted");
                     }
                     Err(err) => {
@@ -115,16 +129,26 @@ impl Imap {
             }
         }
 
-        Ok(())
+        Ok(info)
     }
 
-    pub(crate) async fn fake_idle(&mut self, context: &Context, watch_folder: Option<String>) {
+    pub(crate) async fn fake_idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> InterruptInfo {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
         let fake_idle_start_time = SystemTime::now();
         info!(context, "IMAP-fake-IDLEing...");
 
+        // Do not poll, just wait for an interrupt when no folder is passed in.
+        if watch_folder.is_none() {
+            return self.idle_interrupt.recv().await.unwrap_or_default();
+        }
+
+        let mut info: InterruptInfo = Default::default();
         if self.skip_next_idle_wait {
             // interrupt_idle has happened before we
             // provided self.interrupt
@@ -135,53 +159,61 @@ impl Imap {
             // TODO: grow sleep durations / make them more flexible
             let mut interval = async_std::stream::interval(Duration::from_secs(60));
 
+            enum Event {
+                Tick,
+                Interrupt(InterruptInfo),
+            }
             // loop until we are interrupted or if we fetched something
-            loop {
-                use futures::future::FutureExt;
-                match interval
-                    .next()
-                    .race(self.idle_interrupt.recv().map(|_| None))
-                    .await
-                {
-                    Some(_) => {
-                        // try to connect with proper login params
-                        // (setup_handle_if_needed might not know about them if we
-                        // never successfully connected)
-                        if let Err(err) = self.connect_configured(context).await {
-                            warn!(context, "fake_idle: could not connect: {}", err);
-                            continue;
-                        }
-                        if self.config.can_idle {
-                            // we only fake-idled because network was gone during IDLE, probably
-                            break;
-                        }
-                        info!(context, "fake_idle is connected");
-                        // we are connected, let's see if fetching messages results
-                        // in anything.  If so, we behave as if IDLE had data but
-                        // will have already fetched the messages so perform_*_fetch
-                        // will not find any new.
+            info =
+                loop {
+                    use futures::future::FutureExt;
+                    match interval
+                        .next()
+                        .map(|_| Event::Tick)
+                        .race(self.idle_interrupt.recv().map(|probe_network| {
+                            Event::Interrupt(probe_network.unwrap_or_default())
+                        }))
+                        .await
+                    {
+                        Event::Tick => {
+                            // try to connect with proper login params
+                            // (setup_handle_if_needed might not know about them if we
+                            // never successfully connected)
+                            if let Err(err) = self.connect_configured(context).await {
+                                warn!(context, "fake_idle: could not connect: {}", err);
+                                continue;
+                            }
+                            if self.config.can_idle {
+                                // we only fake-idled because network was gone during IDLE, probably
+                                break InterruptInfo::new(false, None);
+                            }
+                            info!(context, "fake_idle is connected");
+                            // we are connected, let's see if fetching messages results
+                            // in anything.  If so, we behave as if IDLE had data but
+                            // will have already fetched the messages so perform_*_fetch
+                            // will not find any new.
 
-                        if let Some(ref watch_folder) = watch_folder {
-                            match self.fetch_new_messages(context, watch_folder).await {
-                                Ok(res) => {
-                                    info!(context, "fetch_new_messages returned {:?}", res);
-                                    if res {
-                                        break;
+                            if let Some(ref watch_folder) = watch_folder {
+                                match self.fetch_new_messages(context, watch_folder).await {
+                                    Ok(res) => {
+                                        info!(context, "fetch_new_messages returned {:?}", res);
+                                        if res {
+                                            break InterruptInfo::new(false, None);
+                                        }
+                                    }
+                                    Err(err) => {
+                                        error!(context, "could not fetch from folder: {}", err);
+                                        self.trigger_reconnect()
                                     }
-                                }
-                                Err(err) => {
-                                    error!(context, "could not fetch from folder: {}", err);
-                                    self.trigger_reconnect()
                                 }
                             }
                         }
+                        Event::Interrupt(info) => {
+                            // Interrupt
+                            break info;
+                        }
                     }
-                    None => {
-                        // Interrupt
-                        break;
-                    }
-                }
-            }
+                };
         }
 
         info!(
@@ -193,5 +225,7 @@ impl Imap {
                 .as_millis() as f64
                 / 1000.,
         );
+
+        info
     }
 }

src/imap/mod.rs

@@ -3,7 +3,7 @@
 //! uses [async-email/async-imap](https://github.com/async-email/async-imap)
 //! to implement connect, fetch, delete functionality with standard IMAP servers.
 
-use num_traits::FromPrimitive;
+use std::collections::BTreeMap;
 
 use async_imap::{
     error::Result as ImapResult,
@@ -11,6 +11,7 @@ use async_imap::{
 };
 use async_std::prelude::*;
 use async_std::sync::Receiver;
+use num_traits::FromPrimitive;
 
 use crate::config::*;
 use crate::constants::*;
@@ -26,7 +27,8 @@ use crate::message::{self, update_server_uid};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
-use crate::stock::StockMessage;
+use crate::provider::get_provider_info;
+use crate::{scheduler::InterruptInfo, stock::StockMessage};
 
 mod client;
 mod idle;
@@ -108,7 +110,7 @@ const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
-    idle_interrupt: Receiver<()>,
+    idle_interrupt: Receiver<InterruptInfo>,
     config: ImapConfig,
     session: Option<Session>,
     connected: bool,
@@ -148,7 +150,7 @@ struct ImapConfig {
     pub imap_port: u16,
     pub imap_user: String,
     pub imap_pw: String,
-    pub certificate_checks: CertificateChecks,
+    pub strict_tls: bool,
     pub server_flags: usize,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
@@ -168,7 +170,7 @@ impl Default for ImapConfig {
             imap_port: 0,
             imap_user: "".into(),
             imap_pw: "".into(),
-            certificate_checks: Default::default(),
+            strict_tls: false,
             server_flags: 0,
             selected_folder: None,
             selected_mailbox: None,
@@ -180,7 +182,7 @@ impl Default for ImapConfig {
 }
 
 impl Imap {
-    pub fn new(idle_interrupt: Receiver<()>) -> Self {
+    pub fn new(idle_interrupt: Receiver<InterruptInfo>) -> Self {
         Imap {
             idle_interrupt,
             config: Default::default(),
@@ -227,7 +229,7 @@ impl Imap {
                 match Client::connect_insecure((imap_server, imap_port)).await {
                     Ok(client) => {
                         if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.certificate_checks).await
+                            client.secure(imap_server, config.strict_tls).await
                         } else {
                             Ok(client)
                         }
@@ -239,12 +241,8 @@ impl Imap {
                 let imap_server: &str = config.imap_server.as_ref();
                 let imap_port = config.imap_port;
 
-                Client::connect_secure(
-                    (imap_server, imap_port),
-                    imap_server,
-                    config.certificate_checks,
-                )
-                .await
+                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
+                    .await
             };
 
         let login_res = match connection_res {
@@ -294,6 +292,8 @@ impl Imap {
 
         match login_res {
             Ok(session) => {
+                // needs to be set here to ensure it is set on reconnects.
+                self.connected = true;
                 self.session = Some(session);
                 Ok(())
             }
@@ -314,9 +314,15 @@ impl Imap {
     }
 
     async fn unsetup_handle(&mut self, context: &Context) {
+        // Close folder if messages should be expunged
+        if let Err(err) = self.close_folder(context).await {
+            warn!(context, "failed to close folder: {:?}", err);
+        }
+
+        // Logout from the server
         if let Some(mut session) = self.session.take() {
-            if let Err(err) = session.close().await {
-                warn!(context, "failed to close connection: {:?}", err);
+            if let Err(err) = session.logout().await {
+                warn!(context, "failed to logout: {:?}", err);
             }
         }
         self.connected = false;
@@ -376,7 +382,15 @@ impl Imap {
             config.imap_port = imap_port;
             config.imap_user = imap_user.to_string();
             config.imap_pw = imap_pw.to_string();
-            config.certificate_checks = lp.imap_certificate_checks;
+            let provider = get_provider_info(&lp.addr);
+            config.strict_tls = match lp.imap_certificate_checks {
+                CertificateChecks::Automatic => {
+                    provider.map_or(false, |provider| provider.strict_tls)
+                }
+                CertificateChecks::Strict => true,
+                CertificateChecks::AcceptInvalidCertificates
+                | CertificateChecks::AcceptInvalidCertificates2 => false,
+            };
             config.server_flags = server_flags;
         }
 
@@ -577,164 +591,120 @@ impl Imap {
             .select_with_uidvalidity(context, folder.as_ref())
             .await?;
 
-        let mut read_cnt: usize = 0;
-
-        if self.session.is_none() {
-            return Err(Error::NoConnection);
-        }
-        let session = self.session.as_mut().unwrap();
-
-        // fetch messages with larger UID than the last one seen
-        // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
-        let set = format!("{}:*", last_seen_uid + 1);
-
-        info!(context, "fetch_new_messages {:?}", set);
-        let mut list = match session.uid_fetch(&set, PREFETCH_FLAGS).await {
-            Ok(list) => list,
-            Err(err) => {
-                warn!(context, "ERROR: fetch_new_messages {:?} -> {:?}", &set, err);
-                return Err(Error::FetchFailed(err));
-            }
-        };
-        info!(context, "fetch_new_messages {:?} RETURNED", &set);
-
-        let mut msgs = Vec::new();
-        while let Some(fetch) = list.next().await {
-            let fetch = fetch.map_err(|err| Error::Other(err.to_string()))?;
-            msgs.push(fetch);
-        }
-        drop(list);
-        info!(context, "fetch_new_messages got {:?} messsages", msgs.len());
-
-        msgs.sort_unstable_by_key(|msg| msg.uid.unwrap_or_default());
-        let msgs: Vec<_> = msgs
-            .into_iter()
-            .filter(|msg| {
-                let cur_uid = msg.uid.unwrap_or_default();
-                if cur_uid <= last_seen_uid {
-                    // If the mailbox is not empty, results always include
-                    // at least one UID, even if last_seen_uid+1 is past
-                    // the last UID in the mailbox.  It happens because
-                    // uid+1:* is interpreted the same way as *:uid+1.
-                    // See https://tools.ietf.org/html/rfc3501#page-61 for
-                    // standard reference. Therefore, sometimes we receive
-                    // already seen messages and have to filter them out.
-                    info!(
-                        context,
-                        "fetch_new_messages: ignoring uid {}, last seen was {}",
-                        cur_uid,
-                        last_seen_uid
-                    );
-                    false
-                } else {
-                    true
-                }
-            })
-            .collect();
-
-        read_cnt += msgs.len();
+        let msgs = self.fetch_after(context, last_seen_uid).await?;
+        let read_cnt = msgs.len();
+        let folder: &str = folder.as_ref();
 
         let mut read_errors = 0;
-
         let mut uids = Vec::with_capacity(msgs.len());
         let mut new_last_seen_uid = None;
 
-        for fetch in msgs.into_iter() {
-            let folder: &str = folder.as_ref();
-
-            let cur_uid = fetch.uid.unwrap_or_default();
-            let headers = match get_fetch_headers(&fetch) {
-                Ok(h) => h,
+        for (current_uid, msg) in msgs.into_iter() {
+            let (headers, msg_id) = match get_fetch_headers(&msg) {
+                Ok(headers) => {
+                    let msg_id = prefetch_get_message_id(&headers).unwrap_or_default();
+                    (headers, msg_id)
+                }
                 Err(err) => {
-                    warn!(context, "get_fetch_headers error: {}", err);
+                    warn!(context, "{}", err);
                     read_errors += 1;
                     continue;
                 }
             };
 
-            let message_id = prefetch_get_message_id(&headers).unwrap_or_default();
-            let skip = match precheck_imf(context, &message_id, folder, cur_uid).await {
-                Ok(skip) => skip,
-                Err(err) => {
-                    warn!(context, "precheck_imf error: {}", err);
-                    true
-                }
-            };
-
-            if skip {
-                // we know the message-id already or don't want the message otherwise.
-                info!(
-                    context,
-                    "Skipping message {} from \"{}\" by precheck.", message_id, folder,
-                );
-                if read_errors == 0 {
-                    new_last_seen_uid = Some(cur_uid);
-                }
-            } else {
-                // we do not know the message-id
-                // or the message-id is missing (in this case, we create one in the further process)
-                // or some other error happened
-                let show = match prefetch_should_download(context, &headers, show_emails).await {
-                    Ok(show) => show,
-                    Err(err) => {
-                        warn!(context, "prefetch_should_download error: {}", err);
-                        true
-                    }
-                };
-
-                if show {
-                    uids.push(cur_uid);
-                } else {
-                    info!(
-                        context,
-                        "Ignoring new message {} from \"{}\".", message_id, folder,
-                    );
-                }
-                if read_errors == 0 {
-                    new_last_seen_uid = Some(cur_uid);
-                }
+            if message_needs_processing(
+                context,
+                current_uid,
+                &headers,
+                &msg_id,
+                folder,
+                show_emails,
+            )
+            .await
+            {
+                // Trigger download and processing for this message.
+                uids.push(current_uid);
+            } else if read_errors == 0 {
+                // No errors so far, but this was skipped, so mark as last_seen_uid
+                new_last_seen_uid = Some(current_uid);
             }
         }
 
-        info!(
-            context,
-            "fetch_many_msgs fetching {} messages in batch",
-            uids.len()
-        );
         // check passed, go fetch the emails
         let (new_last_seen_uid_processed, error_cnt) =
             self.fetch_many_msgs(context, &folder, &uids).await;
+        read_errors += error_cnt;
 
+        // determine which last_seen_uid to use to update  to
         let new_last_seen_uid_processed = new_last_seen_uid_processed.unwrap_or_default();
         let new_last_seen_uid = new_last_seen_uid.unwrap_or_default();
         let last_one = new_last_seen_uid.max(new_last_seen_uid_processed);
+
         if last_one > last_seen_uid {
-            self.set_config_last_seen_uid(context, &folder, uid_validity, new_last_seen_uid)
+            self.set_config_last_seen_uid(context, &folder, uid_validity, last_one)
                 .await;
         }
 
-        read_errors += error_cnt;
-
-        if read_errors > 0 {
+        if read_errors == 0 {
+            info!(context, "{} mails read from \"{}\".", read_cnt, folder,);
+        } else {
             warn!(
                 context,
-                "{} mails read from \"{}\" with {} errors.",
-                read_cnt,
-                folder.as_ref(),
-                read_errors
-            );
-        } else {
-            info!(
-                context,
-                "{} mails read from \"{}\".",
-                read_cnt,
-                folder.as_ref()
+                "{} mails read from \"{}\" with {} errors.", read_cnt, folder, read_errors
             );
         }
 
         Ok(read_cnt > 0)
     }
 
+    /// Fetch all uids larger than the passed in. Returns a sorted list of fetch results.
+    async fn fetch_after(
+        &mut self,
+        context: &Context,
+        uid: u32,
+    ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
+        if self.session.is_none() {
+            return Err(Error::NoConnection);
+        }
+
+        let session = self.session.as_mut().unwrap();
+
+        // fetch messages with larger UID than the last one seen
+        // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
+        let set = format!("{}:*", uid + 1);
+        let mut list = session
+            .uid_fetch(set, PREFETCH_FLAGS)
+            .await
+            .map_err(Error::FetchFailed)?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(fetch) = list.next().await {
+            let msg = fetch.map_err(|err| Error::Other(err.to_string()))?;
+            if let Some(msg_uid) = msg.uid {
+                msgs.insert(msg_uid, msg);
+            }
+        }
+        drop(list);
+
+        // If the mailbox is not empty, results always include
+        // at least one UID, even if last_seen_uid+1 is past
+        // the last UID in the mailbox.  It happens because
+        // uid+1:* is interpreted the same way as *:uid+1.
+        // See https://tools.ietf.org/html/rfc3501#page-61 for
+        // standard reference. Therefore, sometimes we receive
+        // already seen messages and have to filter them out.
+        let new_msgs = msgs.split_off(&(uid + 1));
+
+        for current_uid in msgs.keys() {
+            info!(
+                context,
+                "fetch_new_messages: ignoring uid {}, last seen was {}", current_uid, uid
+            );
+        }
+
+        Ok(new_msgs)
+    }
+
     async fn set_config_last_seen_uid<S: AsRef<str>>(
         &self,
         context: &Context,
@@ -766,6 +736,20 @@ impl Imap {
             return (None, 0);
         }
 
+        if !self.is_connected() {
+            warn!(context, "Not connected");
+            return (None, server_uids.len());
+        }
+
+        if self.session.is_none() {
+            // we could not get a valid imap session, this should be retried
+            self.trigger_reconnect();
+            warn!(context, "Could not get IMAP session");
+            return (None, server_uids.len());
+        }
+
+        let session = self.session.as_mut().unwrap();
+
         let set = if server_uids.len() == 1 {
             server_uids[0].to_string()
         } else {
@@ -775,74 +759,67 @@ impl Imap {
             format!("{}:{}", first_uid, last_uid)
         };
 
-        if !self.is_connected() {
-            warn!(context, "Not connected");
-            return (None, server_uids.len());
-        }
-
-        let mut msgs = if let Some(ref mut session) = &mut self.session {
-            match session.uid_fetch(&set, BODY_FLAGS).await {
-                Ok(msgs) => msgs,
-                Err(err) => {
-                    // TODO maybe differentiate between IO and input/parsing problems
-                    // so we don't reconnect if we have a (rare) input/output parsing problem?
-                    self.should_reconnect = true;
-                    warn!(
-                        context,
-                        "Error on fetching messages #{} from folder \"{}\"; error={}.",
-                        &set,
-                        folder.as_ref(),
-                        err
-                    );
-                    return (None, server_uids.len());
-                }
+        let mut msgs = match session.uid_fetch(&set, BODY_FLAGS).await {
+            Ok(msgs) => msgs,
+            Err(err) => {
+                // TODO: maybe differentiate between IO and input/parsing problems
+                // so we don't reconnect if we have a (rare) input/output parsing problem?
+                self.should_reconnect = true;
+                warn!(
+                    context,
+                    "Error on fetching messages #{} from folder \"{}\"; error={}.",
+                    &set,
+                    folder.as_ref(),
+                    err
+                );
+                return (None, server_uids.len());
             }
-        } else {
-            // we could not get a valid imap session, this should be retried
-            self.trigger_reconnect();
-            warn!(context, "Could not get IMAP session");
-            return (None, server_uids.len());
         };
 
+        let folder = folder.as_ref().to_string();
+
         let mut read_errors = 0;
         let mut last_uid = None;
         let mut count = 0;
 
-        let mut jobs = Vec::with_capacity(server_uids.len());
-
+        let mut tasks = Vec::with_capacity(server_uids.len());
         while let Some(Ok(msg)) = msgs.next().await {
             let server_uid = msg.uid.unwrap_or_default();
+
             if !server_uids.contains(&server_uid) {
                 // skip if there are some in between we are not interested in
                 continue;
             }
             count += 1;
 
-            // XXX put flags into a set and pass them to dc_receive_imf
             let is_deleted = msg.flags().any(|flag| flag == Flag::Deleted);
-            let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
+            if is_deleted || msg.body().is_none() {
+                // No need to process these.
+                continue;
+            }
 
-            if !is_deleted && msg.body().is_some() {
-                let folder = folder.as_ref().to_string();
-                let context = context.clone();
-                let task = async_std::task::spawn(async move {
-                    let body = msg.body().unwrap_or_default();
+            // XXX put flags into a set and pass them to dc_receive_imf
+            let context = context.clone();
+            let folder = folder.clone();
 
-                    if let Err(err) =
-                        dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await
-                    {
+            let task = async_std::task::spawn(async move {
+                // safe, as we checked above that there is a body.
+                let body = msg.body().unwrap();
+                let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
+
+                match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
+                    Ok(_) => Some(server_uid),
+                    Err(err) => {
                         warn!(context, "dc_receive_imf error: {}", err);
                         read_errors += 1;
                         None
-                    } else {
-                        Some(server_uid)
                     }
-                });
-                jobs.push(task);
-            }
+                }
+            });
+            tasks.push(task);
         }
 
-        for task in futures::future::join_all(jobs).await {
+        for task in futures::future::join_all(tasks).await {
             match task {
                 Some(uid) => {
                     last_uid = Some(uid);
@@ -1008,7 +985,7 @@ impl Imap {
         uid: u32,
     ) -> Option<ImapActionResult> {
         if uid == 0 {
-            return Some(ImapActionResult::Failed);
+            return Some(ImapActionResult::RetryLater);
         }
         if !self.is_connected() {
             // currently jobs are only performed on the INBOX thread
@@ -1176,54 +1153,54 @@ impl Imap {
                 }
             };
 
+            let mut delimiter = ".".to_string();
+            let mut delimiter_is_default = true;
             let mut sentbox_folder = None;
             let mut mvbox_folder = None;
-
-            let mut delimiter = ".".to_string();
-            if let Some(folder) = folders.next().await {
-                let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
-                if let Some(d) = folder.delimiter() {
-                    if !d.is_empty() {
-                        delimiter = d.to_string();
-                    }
-                }
-            }
-            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
-            let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
+            let mut fallback_folder = get_fallback_folder(&delimiter);
 
             while let Some(folder) = folders.next().await {
                 let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
                 info!(context, "Scanning folder: {:?}", folder);
 
-                if mvbox_folder.is_none()
-                    && (folder.name() == "DeltaChat" || folder.name() == fallback_folder)
-                {
-                    mvbox_folder = Some(folder.name().to_string());
-                }
-
-                if sentbox_folder.is_none() {
-                    if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
-                        sentbox_folder = Some(folder);
-                    } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
-                        sentbox_folder = Some(folder);
+                // Update the delimiter iff there is a different one, but only once.
+                if let Some(d) = folder.delimiter() {
+                    if delimiter_is_default && !d.is_empty() && delimiter != d {
+                        delimiter = d.to_string();
+                        fallback_folder = get_fallback_folder(&delimiter);
+                        delimiter_is_default = false;
                     }
                 }
 
-                if mvbox_folder.is_some() && sentbox_folder.is_some() {
-                    break;
+                if folder.name() == "DeltaChat" {
+                    // Always takes precendent
+                    mvbox_folder = Some(folder.name().to_string());
+                } else if folder.name() == fallback_folder {
+                    // only set iff none has been already set
+                    if mvbox_folder.is_none() {
+                        mvbox_folder = Some(folder.name().to_string());
+                    }
+                } else if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
+                    // Always takes precedent
+                    sentbox_folder = Some(folder.name().to_string());
+                } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
+                    // only set iff none has been already set
+                    if sentbox_folder.is_none() {
+                        sentbox_folder = Some(folder.name().to_string());
+                    }
                 }
             }
-            info!(context, "sentbox folder is {:?}", sentbox_folder);
-
             drop(folders);
 
+            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
+            info!(context, "sentbox folder is {:?}", sentbox_folder);
+
             if mvbox_folder.is_none() && create_mvbox {
                 info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
 
                 match session.create("DeltaChat").await {
                     Ok(_) => {
                         mvbox_folder = Some("DeltaChat".into());
-
                         info!(context, "MVBOX-folder created.",);
                     }
                     Err(err) => {
@@ -1257,23 +1234,16 @@ impl Imap {
                 }
             }
             context
-                .sql
-                .set_raw_config(context, "configured_inbox_folder", Some("INBOX"))
+                .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
                 .await?;
             if let Some(ref mvbox_folder) = mvbox_folder {
                 context
-                    .sql
-                    .set_raw_config(context, "configured_mvbox_folder", Some(mvbox_folder))
+                    .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
                     .await?;
             }
             if let Some(ref sentbox_folder) = sentbox_folder {
                 context
-                    .sql
-                    .set_raw_config(
-                        context,
-                        "configured_sentbox_folder",
-                        Some(sentbox_folder.name()),
-                    )
+                    .set_config(Config::ConfiguredSentboxFolder, Some(sentbox_folder))
                     .await?;
             }
             context
@@ -1431,7 +1401,11 @@ async fn precheck_imf(
         }
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
-            update_server_uid(context, &rfc724_mid, server_folder, server_uid).await;
+            update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
+            context
+                .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
+                .await;
+            info!(context, "Updating server_uid and interrupting")
         }
         Ok(true)
     } else {
@@ -1503,3 +1477,54 @@ async fn prefetch_should_download(
     let show = show && !blocked_contact;
     Ok(show)
 }
+
+async fn message_needs_processing(
+    context: &Context,
+    current_uid: u32,
+    headers: &[mailparse::MailHeader<'_>],
+    msg_id: &str,
+    folder: &str,
+    show_emails: ShowEmails,
+) -> bool {
+    let skip = match precheck_imf(context, &msg_id, folder, current_uid).await {
+        Ok(skip) => skip,
+        Err(err) => {
+            warn!(context, "precheck_imf error: {}", err);
+            true
+        }
+    };
+
+    if skip {
+        // we know the message-id already or don't want the message otherwise.
+        info!(
+            context,
+            "Skipping message {} from \"{}\" by precheck.", msg_id, folder,
+        );
+        return false;
+    }
+
+    // we do not know the message-id
+    // or the message-id is missing (in this case, we create one in the further process)
+    // or some other error happened
+    let show = match prefetch_should_download(context, &headers, show_emails).await {
+        Ok(show) => show,
+        Err(err) => {
+            warn!(context, "prefetch_should_download error: {}", err);
+            true
+        }
+    };
+
+    if !show {
+        info!(
+            context,
+            "Ignoring new message {} from \"{}\".", msg_id, folder,
+        );
+        return false;
+    }
+
+    true
+}
+
+fn get_fallback_folder(delimiter: &str) -> String {
+    format!("INBOX{}DeltaChat", delimiter)
+}

src/imap/select_folder.rs

@@ -27,7 +27,7 @@ impl Imap {
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 

src/imex.rs

@@ -1,5 +1,6 @@
 //! # Import/export module
 
+use std::any::Any;
 use std::cmp::{max, min};
 
 use async_std::path::{Path, PathBuf};
@@ -16,7 +17,7 @@ use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
-use crate::key::{self, DcKey, Key, SignedSecretKey};
+use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -181,13 +182,13 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let private_key = Key::from(SignedSecretKey::load_self(context).await?);
+    let private_key = SignedSecretKey::load_self(context).await?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
+    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -274,7 +275,7 @@ pub async fn continue_key_transfer(
     if let Some(filename) = msg.get_file(context) {
         let file = dc_open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
-        let armored_key = decrypt_setup_file(context, &sc, file)?;
+        let armored_key = decrypt_setup_file(&sc, file).await?;
         set_self_key(context, &armored_key, true, true).await?;
         maybe_add_bcc_self_device_msg(context).await?;
 
@@ -291,12 +292,8 @@ async fn set_self_key(
     prefer_encrypt_required: bool,
 ) -> Result<()> {
     // try hard to only modify key-state
-    let keys = Key::from_armored_string(armored, KeyType::Private)
-        .and_then(|(k, h)| if k.verify() { Some((k, h)) } else { None })
-        .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
-
-    ensure!(keys.is_some(), "Not a valid private key");
-    let (private_key, public_key, header) = keys.unwrap();
+    let (private_key, header) = SignedSecretKey::from_asc(armored)?;
+    let public_key = private_key.split_public_key()?;
     let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
     match preferencrypt.map(|s| s.as_str()) {
         Some(headerval) => {
@@ -322,15 +319,10 @@ async fn set_self_key(
     let self_addr = context.get_config(Config::ConfiguredAddr).await;
     ensure!(self_addr.is_some(), "Missing self addr");
     let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
-
-    let (public, secret) = match (public_key, private_key) {
-        (Key::Public(p), Key::Secret(s)) => (p, s),
-        _ => bail!("wrong keys unpacked"),
-    };
     let keypair = pgp::KeyPair {
         addr,
-        public,
-        secret,
+        public: public_key,
+        secret: private_key,
     };
     key::store_self_keypair(
         context,
@@ -345,12 +337,11 @@ async fn set_self_key(
     Ok(())
 }
 
-fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
-    _context: &Context,
+async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     file: T,
 ) -> Result<String> {
-    let plain_bytes = pgp::symm_decrypt(passphrase, file)?;
+    let plain_bytes = pgp::symm_decrypt(passphrase, file).await?;
     let plain_text = std::string::String::from_utf8(plain_bytes)?;
 
     Ok(plain_text)
@@ -697,9 +688,9 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
             |row| {
                 let id = row.get(0)?;
                 let public_key_blob: Vec<u8> = row.get(1)?;
-                let public_key = Key::from_slice(&public_key_blob, KeyType::Public);
+                let public_key = SignedPublicKey::from_slice(&public_key_blob);
                 let private_key_blob: Vec<u8> = row.get(2)?;
-                let private_key = Key::from_slice(&private_key_blob, KeyType::Private);
+                let private_key = SignedSecretKey::from_slice(&private_key_blob);
                 let is_default: i32 = row.get(3)?;
 
                 Ok((id, public_key, private_key, is_default))
@@ -713,7 +704,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
 
     for (id, public_key, private_key, is_default) in keys {
         let id = Some(id).filter(|_| is_default != 0);
-        if let Some(key) = public_key {
+        if let Ok(key) = public_key {
             if export_key_to_asc_file(context, &dir, id, &key)
                 .await
                 .is_err()
@@ -723,7 +714,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
         } else {
             export_errors += 1;
         }
-        if let Some(key) = private_key {
+        if let Ok(key) = private_key {
             if export_key_to_asc_file(context, &dir, id, &key)
                 .await
                 .is_err()
@@ -742,22 +733,32 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
 /*******************************************************************************
  * Classic key export
  ******************************************************************************/
-async fn export_key_to_asc_file(
+async fn export_key_to_asc_file<T>(
     context: &Context,
     dir: impl AsRef<Path>,
     id: Option<i64>,
-    key: &Key,
-) -> std::io::Result<()> {
+    key: &T,
+) -> std::io::Result<()>
+where
+    T: DcKey + Any,
+{
     let file_name = {
-        let kind = if key.is_public() { "public" } else { "private" };
+        let any_key = key as &dyn Any;
+        let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "public"
+        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "private"
+        } else {
+            "unknown"
+        };
         let id = id.map_or("default".into(), |i| i.to_string());
-
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
     info!(context, "Exporting key {}", file_name.display());
     dc_delete_file(context, &file_name).await;
 
-    let res = key.write_asc_to_file(&file_name, context).await;
+    let content = key.to_asc(None).into_bytes();
+    let res = dc_write_file(context, &file_name, &content).await;
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
@@ -823,7 +824,7 @@ mod tests {
     #[async_std::test]
     async fn test_export_key_to_asc_file() {
         let context = dummy_context().await;
-        let key = Key::from(alice_keypair().public);
+        let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
             .await
@@ -852,9 +853,6 @@ mod tests {
 
     #[async_std::test]
     async fn test_split_and_decrypt() {
-        let ctx = dummy_context().await;
-        let context = &ctx.ctx;
-
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
         let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
         assert_eq!(typ, BlockType::Message);
@@ -864,12 +862,10 @@ mod tests {
         assert!(!base64.is_empty());
 
         let setup_file = S_EM_SETUPFILE.to_string();
-        let decrypted = decrypt_setup_file(
-            context,
-            S_EM_SETUPCODE,
-            std::io::Cursor::new(setup_file.as_bytes()),
-        )
-        .unwrap();
+        let decrypted =
+            decrypt_setup_file(S_EM_SETUPCODE, std::io::Cursor::new(setup_file.as_bytes()))
+                .await
+                .unwrap();
 
         let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
 

src/job.rs

@@ -3,6 +3,7 @@
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
 
+use std::env;
 use std::fmt;
 use std::future::Future;
 
@@ -31,7 +32,8 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
-use crate::sql;
+use crate::upload::{download_message_file, generate_upload_url, upload_file};
+use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
@@ -107,6 +109,8 @@ pub enum Action {
     MaybeSendLocationsEnded = 5007,
     SendMdn = 5010,
     SendMsgToSmtp = 5901, // ... high priority
+
+    DownloadMessageFile = 7000,
 }
 
 impl Default for Action {
@@ -133,6 +137,9 @@ impl From<Action> for Thread {
             MaybeSendLocationsEnded => Thread::Smtp,
             SendMdn => Thread::Smtp,
             SendMsgToSmtp => Thread::Smtp,
+
+            // TODO: Where does downloading fit in the thread architecture?
+            DownloadMessageFile => Thread::Imap,
         }
     }
 }
@@ -190,7 +197,7 @@ impl Job {
     /// Saves the job to the database, creating a new entry if necessary.
     ///
     /// The Job is consumed by this method.
-    pub async fn save(self, context: &Context) -> Result<()> {
+    pub(crate) async fn save(self, context: &Context) -> Result<()> {
         let thread: Thread = self.action.into();
 
         info!(context, "saving job for {}-thread: {:?}", thread, self);
@@ -329,7 +336,15 @@ impl Job {
         }
     }
 
-    pub async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+        // Upload file to HTTP if set in params.
+        if let (Some(upload_url), Ok(Some(upload_path))) = (
+            self.param.get_upload_url(),
+            self.param.get_upload_path(context),
+        ) {
+            job_try!(upload_file(context, upload_url.to_string(), upload_path).await);
+        }
+
         //  SMTP server, if not yet done
         if !smtp.is_connected().await {
             let loginparam = LoginParam::from_database(context, "configured_").await;
@@ -498,16 +513,13 @@ impl Job {
     }
 
     async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-
-        if let Err(err) = imap.ensure_configured_folders(context, true).await {
-            warn!(context, "could not configure folders: {:?}", err);
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
-        let dest_folder = context
-            .sql
-            .get_raw_config(context, "configured_mvbox_folder")
-            .await;
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
 
         if let Some(dest_folder) = dest_folder {
             let server_folder = msg.server_folder.as_ref().unwrap();
@@ -518,7 +530,7 @@ impl Job {
             {
                 ImapActionResult::RetryLater => Status::RetryLater,
                 ImapActionResult::Success => {
-                    // XXX Rust-Imap provides no target uid on mv, so just set it to 0
+                    // Rust-Imap provides no target uid on mv, so just set it to 0, update again when precheck_imf() is called for the moved message
                     message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await;
                     Status::Finished(Ok(()))
                 }
@@ -541,6 +553,11 @@ impl Job {
     /// records pointing to the same message on the server, the job
     /// also removes the message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         if !msg.rfc724_mid.is_empty() {
@@ -611,12 +628,13 @@ impl Job {
     }
 
     async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = context
-                .sql
-                .get_raw_config(context, "configured_mvbox_folder")
-                .await
-            {
+            if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
                 imap.empty_folder(context, &mvbox_folder).await;
             }
         }
@@ -627,6 +645,11 @@ impl Job {
     }
 
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
@@ -650,6 +673,13 @@ impl Job {
             }
         }
     }
+
+    pub(crate) async fn download_message_file(&mut self, context: &Context) -> Status {
+        let msg_id = MsgId::new(self.foreign_id);
+        let download_path = job_try!(self.param.get_upload_path(context));
+        job_try!(download_message_file(context, msg_id, download_path).await);
+        Status::Finished(Ok(()))
+    }
 }
 
 /// Delete all pending jobs with the given action.
@@ -718,7 +748,23 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
         }
     };
 
-    let mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
+    let mut mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
+
+    // Prepare file upload if DCC_UPLOAD_URL env variable is set.
+    // See upload-server folder for an example server impl.
+    // Here a new URL is generated, which the mimefactory includes in the message instead of the
+    // actual attachement. The upload then happens in the smtp send job.
+    let upload = if let Some(file) = msg.get_file(context) {
+        if let Ok(endpoint) = env::var("DCC_UPLOAD_URL") {
+            let upload_url = generate_upload_url(context, endpoint);
+            mimefactory.set_upload_url(upload_url.clone());
+            Some((upload_url, file))
+        } else {
+            None
+        }
+    } else {
+        None
+    };
 
     let mut recipients = mimefactory.recipients();
 
@@ -810,13 +856,17 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     param.set(Param::File, blob.as_name());
     param.set(Param::Recipients, &recipients);
 
+    if let Some((upload_url, upload_path)) = upload {
+        param.set_upload_url(upload_url);
+        param.set_upload_path(upload_path);
+    }
+
     let job = create(Action::SendMsgToSmtp, msg_id.to_u32() as i32, param, 0)?;
 
     Ok(Some(job))
 }
 
-#[derive(Debug)]
-pub enum Connection<'a> {
+pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
@@ -971,6 +1021,7 @@ async fn perform_job_action(
             sql::housekeeping(context).await;
             Status::Finished(Ok(()))
         }
+        Action::DownloadMessageFile => job.download_message_file(context).await,
     };
 
     info!(
@@ -1027,14 +1078,21 @@ pub async fn add(context: &Context, job: Job) {
             | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
             | Action::MarkseenMsgOnImap
-            | Action::MoveMsg => {
-                context.interrupt_inbox().await;
+            | Action::MoveMsg
+            | Action::DownloadMessageFile => {
+                info!(context, "interrupt: imap");
+                context
+                    .interrupt_inbox(InterruptInfo::new(false, None))
+                    .await;
             }
             Action::MaybeSendLocations
             | Action::MaybeSendLocationsEnded
             | Action::SendMdn
             | Action::SendMsgToSmtp => {
-                context.interrupt_smtp().await;
+                info!(context, "interrupt: smtp");
+                context
+                    .interrupt_smtp(InterruptInfo::new(false, None))
+                    .await;
             }
         }
     }
@@ -1049,38 +1107,49 @@ pub async fn add(context: &Context, job: Job) {
 pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
-    probe_network: bool,
+    info: &InterruptInfo,
 ) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
-    let query = if !probe_network {
+
+    let query;
+    let params;
+    let t = time();
+    let m;
+    let thread_i = thread as i64;
+
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
+SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+FROM jobs
+WHERE thread=? AND foreign_id=?
+ORDER BY action DESC, added_timestamp
+LIMIT 1;
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
         // processing for first-try and after backoff-timeouts:
         // process jobs in the order they were added.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND desired_timestamp<=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#
+"#;
+        params = paramsv![thread_i, t];
     } else {
         // processing after call to dc_maybe_network():
         // process _all_ pending jobs that failed before
         // in the order of their backoff-times.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND tries>0
 ORDER BY desired_timestamp, action DESC
 LIMIT 1;
-"#
-    };
-
-    let thread_i = thread as i64;
-    let t = time();
-    let params = if !probe_network {
-        paramsv![thread_i, t]
-    } else {
-        paramsv![thread_i]
+"#;
+        params = paramsv![thread_i];
     };
 
     let job = loop {
@@ -1088,13 +1157,13 @@ LIMIT 1;
             .sql
             .query_row_optional(query, params.clone(), |row| {
                 let job = Job {
-                    job_id: row.get(0)?,
-                    action: row.get(1)?,
-                    foreign_id: row.get(2)?,
-                    desired_timestamp: row.get(5)?,
-                    added_timestamp: row.get(4)?,
-                    tries: row.get(6)?,
-                    param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
+                    job_id: row.get("id")?,
+                    action: row.get("action")?,
+                    foreign_id: row.get("foreign_id")?,
+                    desired_timestamp: row.get("desired_timestamp")?,
+                    added_timestamp: row.get("added_timestamp")?,
+                    tries: row.get("tries")?,
+                    param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
                     pending_error: None,
                 };
 
@@ -1104,8 +1173,9 @@ LIMIT 1;
 
         match job_res {
             Ok(job) => break job,
-            Err(_) => {
+            Err(err) => {
                 // Remove invalid job from the DB
+                info!(context, "cleaning up job, because of {}", err);
 
                 // TODO: improve by only doing a single query
                 match context
@@ -1116,7 +1186,7 @@ LIMIT 1;
                     Ok(id) => {
                         context
                             .sql
-                            .execute("DELETE FROM jobs WHERE id=?", paramsv![id])
+                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
                             .await
                             .ok();
                     }
@@ -1129,21 +1199,26 @@ LIMIT 1;
         }
     };
 
-    if thread == Thread::Imap {
-        if let Some(job) = job {
-            if job.action < Action::DeleteMsgOnImap {
-                load_imap_deletion_job(context)
-                    .await
-                    .unwrap_or_default()
-                    .or(Some(job))
-            } else {
-                Some(job)
-            }
-        } else {
-            load_imap_deletion_job(context).await.unwrap_or_default()
+    match thread {
+        Thread::Unknown => {
+            error!(context, "unknown thread for job");
+            None
         }
-    } else {
-        job
+        Thread::Imap => {
+            if let Some(job) = job {
+                if job.action < Action::DeleteMsgOnImap {
+                    load_imap_deletion_job(context)
+                        .await
+                        .unwrap_or_default()
+                        .or(Some(job))
+                } else {
+                    Some(job)
+                }
+            } else {
+                load_imap_deletion_job(context).await.unwrap_or_default()
+            }
+        }
+        Thread::Smtp => job,
     }
 }
 
@@ -1175,17 +1250,42 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_load_next_job() {
+    async fn test_load_next_job_two() {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
         let t = dummy_context().await;
         insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_none());
 
         insert_job(&t.ctx, 1).await;
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
+        assert!(jobs.is_some());
+    }
+
+    #[async_std::test]
+    async fn test_load_next_job_one() {
+        let t = dummy_context().await;
+
+        insert_job(&t.ctx, 1).await;
+
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_some());
     }
 }

src/key.rs

@@ -1,19 +1,20 @@
 //! Cryptographic key module
 
 use std::collections::BTreeMap;
+use std::fmt;
 use std::io::Cursor;
 
-use async_std::path::Path;
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
+use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 use crate::sql;
 
 // Re-export key types
@@ -38,6 +39,8 @@ pub enum Error {
     NoConfiguredAddr,
     #[error("Configured address is invalid: {}", _0)]
     InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[error("no data provided")]
+    Empty,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -48,8 +51,8 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
 #[async_trait]
-pub trait DcKey: Serialize + Deserializable {
-    type KeyType: Serialize + Deserializable;
+pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
+    type KeyType: Serialize + Deserializable + KeyTrait + Clone;
 
     /// Create a key from some bytes.
     fn from_slice(bytes: &[u8]) -> Result<Self::KeyType> {
@@ -66,18 +69,45 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
+    /// Create a key from an ASCII-armored string.
+    ///
+    /// Returns the key and a map of any headers which might have been set in
+    /// the ASCII-armored representation.
+    fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
+        let bytes = data.as_bytes();
+        Self::KeyType::from_armor_single(Cursor::new(bytes)).map_err(Error::Pgp)
+    }
+
     /// Load the users' default key from the database.
     async fn load_self(context: &Context) -> Result<Self::KeyType>;
 
-    /// Serialise the key to a base64 string.
-    fn to_base64(&self) -> String {
+    /// Serialise the key as bytes.
+    fn to_bytes(&self) -> Vec<u8> {
         // Not using Serialize::to_bytes() to make clear *why* it is
         // safe to ignore this error.
         // Because we write to a Vec<u8> the io::Write impls never
         // fail and we can hide this error.
         let mut buf = Vec::new();
         self.to_writer(&mut buf).unwrap();
-        base64::encode(&buf)
+        buf
+    }
+
+    /// Serialise the key to a base64 string.
+    fn to_base64(&self) -> String {
+        base64::encode(&DcKey::to_bytes(self))
+    }
+
+    /// Serialise the key to ASCII-armored representation.
+    ///
+    /// Each header line must be terminated by `\r\n`.  Only allows setting one
+    /// header as a simplification since that's the only way it's used so far.
+    // Since .to_armored_string() are actual methods on SignedPublicKey and
+    // SignedSecretKey we can not generically implement this.
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String;
+
+    /// The fingerprint for the key.
+    fn fingerprint(&self) -> Fingerprint {
+        Fingerprint::new(KeyTrait::fingerprint(self)).expect("Invalid fingerprint from rpgp")
     }
 }
 
@@ -108,6 +138,22 @@ impl DcKey for SignedPublicKey {
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to ignore this error.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
 }
 
 #[async_trait]
@@ -137,6 +183,39 @@ impl DcKey for SignedSecretKey {
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to do these unwraps.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.  The string is always ASCII.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
+}
+
+/// Deltachat extension trait for secret keys.
+///
+/// Provides some convenience wrappers only applicable to [SignedSecretKey].
+pub trait DcSecretKey {
+    /// Create a public key from a private one.
+    fn split_public_key(&self) -> Result<SignedPublicKey>;
+}
+
+impl DcSecretKey for SignedSecretKey {
+    fn split_public_key(&self) -> Result<SignedPublicKey> {
+        self.verify()?;
+        let unsigned_pubkey = SecretKeyTrait::public_key(self);
+        let signed_pubkey = unsigned_pubkey.sign(self, || "".into())?;
+        Ok(signed_pubkey)
+    }
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
@@ -172,7 +251,9 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
-            let keypair = crate::pgp::create_keypair(addr, keytype)?;
+            let keypair =
+                async_std::task::spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
+                    .await?;
             store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
             info!(
                 context,
@@ -185,198 +266,6 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     }
 }
 
-/// Cryptographic key
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum Key {
-    Public(SignedPublicKey),
-    Secret(SignedSecretKey),
-}
-
-impl From<SignedPublicKey> for Key {
-    fn from(key: SignedPublicKey) -> Self {
-        Key::Public(key)
-    }
-}
-
-impl From<SignedSecretKey> for Key {
-    fn from(key: SignedSecretKey) -> Self {
-        Key::Secret(key)
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl Key {
-    pub fn is_public(&self) -> bool {
-        match self {
-            Key::Public(_) => true,
-            Key::Secret(_) => false,
-        }
-    }
-
-    pub fn is_secret(&self) -> bool {
-        !self.is_public()
-    }
-
-    pub fn from_slice(bytes: &[u8], key_type: KeyType) -> Option<Self> {
-        if bytes.is_empty() {
-            return None;
-        }
-        let res: std::result::Result<Key, _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-            KeyType::Private => SignedSecretKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-        };
-
-        match res {
-            Ok(key) => Some(key),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn from_armored_string(
-        data: &str,
-        key_type: KeyType,
-    ) -> Option<(Self, BTreeMap<String, String>)> {
-        let bytes = data.as_bytes();
-        let res: std::result::Result<(Key, _), _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-            KeyType::Private => SignedSecretKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-        };
-
-        match res {
-            Ok(res) => Some(res),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn to_bytes(&self) -> Vec<u8> {
-        match self {
-            Key::Public(k) => k.to_bytes().unwrap_or_default(),
-            Key::Secret(k) => k.to_bytes().unwrap_or_default(),
-        }
-    }
-
-    pub fn verify(&self) -> bool {
-        match self {
-            Key::Public(k) => k.verify().is_ok(),
-            Key::Secret(k) => k.verify().is_ok(),
-        }
-    }
-
-    pub fn to_base64(&self) -> String {
-        let buf = self.to_bytes();
-        base64::encode(&buf)
-    }
-
-    pub fn to_armored_string(
-        &self,
-        headers: Option<&BTreeMap<String, String>>,
-    ) -> pgp::errors::Result<String> {
-        match self {
-            Key::Public(k) => k.to_armored_string(headers),
-            Key::Secret(k) => k.to_armored_string(headers),
-        }
-    }
-
-    /// Each header line must be terminated by `\r\n`
-    pub fn to_asc(&self, header: Option<(&str, &str)>) -> String {
-        let headers = header.map(|(key, value)| {
-            let mut m = BTreeMap::new();
-            m.insert(key.to_string(), value.to_string());
-            m
-        });
-
-        self.to_armored_string(headers.as_ref())
-            .expect("failed to serialize key")
-    }
-
-    pub async fn write_asc_to_file(
-        &self,
-        file: impl AsRef<Path>,
-        context: &Context,
-    ) -> std::io::Result<()> {
-        let file_content = self.to_asc(None).into_bytes();
-
-        let res = dc_write_file(context, &file, &file_content).await;
-        if res.is_err() {
-            error!(context, "Cannot write key to {}", file.as_ref().display());
-        }
-        res
-    }
-
-    pub fn fingerprint(&self) -> String {
-        match self {
-            Key::Public(k) => hex::encode_upper(k.fingerprint()),
-            Key::Secret(k) => hex::encode_upper(k.fingerprint()),
-        }
-    }
-
-    pub fn formatted_fingerprint(&self) -> String {
-        let rawhex = self.fingerprint();
-        dc_format_fingerprint(&rawhex)
-    }
-
-    pub fn split_key(&self) -> Option<Key> {
-        match self {
-            Key::Public(_) => None,
-            Key::Secret(k) => {
-                let pub_key = k.public_key();
-                pub_key.sign(k, || "".into()).map(Key::Public).ok()
-            }
-        }
-    }
-}
-
 /// Use of a [KeyPair] for encryption or decryption.
 ///
 /// This is used by [store_self_keypair] to know what kind of key is
@@ -426,14 +315,8 @@ pub async fn store_self_keypair(
 ) -> std::result::Result<(), SaveKeyError> {
     // Everything should really be one transaction, more refactoring
     // is needed for that.
-    let public_key = keypair
-        .public
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise public key", err))?;
-    let secret_key = keypair
-        .secret
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise secret key", err))?;
+    let public_key = DcKey::to_bytes(&keypair.public);
+    let secret_key = DcKey::to_bytes(&keypair.secret);
     context
         .sql
         .execute(
@@ -471,37 +354,73 @@ pub async fn store_self_keypair(
     Ok(())
 }
 
-/// Make a fingerprint human-readable, in hex format.
-pub fn dc_format_fingerprint(fingerprint: &str) -> String {
-    // split key into chunks of 4 with space, and 20 newline
-    let mut res = String::new();
+/// A key fingerprint
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct Fingerprint(Vec<u8>);
 
-    for (i, c) in fingerprint.chars().enumerate() {
-        if i > 0 && i % 20 == 0 {
-            res += "\n";
-        } else if i > 0 && i % 4 == 0 {
-            res += " ";
+impl Fingerprint {
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+        match v.len() {
+            20 => Ok(Fingerprint(v)),
+            _ => Err(FingerprintError::WrongLength),
         }
-
-        res += &c.to_string();
     }
 
-    res
+    /// Make a hex string from the fingerprint.
+    ///
+    /// Use [std::fmt::Display] or [ToString::to_string] to get a
+    /// human-readable formatted string.
+    pub fn hex(&self) -> String {
+        hex::encode_upper(&self.0)
+    }
 }
 
-/// Bring a human-readable or otherwise formatted fingerprint back to the 40-characters-uppercase-hex format.
-pub fn dc_normalize_fingerprint(fp: &str) -> String {
-    fp.to_uppercase()
-        .chars()
-        .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
-        .collect()
+/// Make a human-readable fingerprint.
+impl fmt::Display for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Split key into chunks of 4 with space and newline at 20 chars
+        for (i, c) in self.hex().chars().enumerate() {
+            if i > 0 && i % 20 == 0 {
+                writeln!(f)?;
+            } else if i > 0 && i % 4 == 0 {
+                write!(f, " ")?;
+            }
+            write!(f, "{}", c)?;
+        }
+        Ok(())
+    }
+}
+
+/// Parse a human-readable or otherwise formatted fingerprint.
+impl std::str::FromStr for Fingerprint {
+    type Err = FingerprintError;
+
+    fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
+        let hex_repr: String = input
+            .to_uppercase()
+            .chars()
+            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .collect();
+        let v: Vec<u8> = hex::decode(hex_repr)?;
+        let fp = Fingerprint::new(v)?;
+        Ok(fp)
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use std::convert::TryFrom;
+
+    use std::error::Error;
 
     use async_std::sync::Arc;
     use lazy_static::lazy_static;
@@ -510,16 +429,9 @@ mod tests {
         static ref KEYPAIR: KeyPair = alice_keypair();
     }
 
-    #[test]
-    fn test_normalize_fingerprint() {
-        let fingerprint = dc_normalize_fingerprint(" 1234  567890 \n AbcD abcdef ABCDEF ");
-
-        assert_eq!(fingerprint, "1234567890ABCDABCDEFABCDEF");
-    }
-
     #[test]
     fn test_from_armored_string() {
-        let (private_key, _) = Key::from_armored_string(
+        let (private_key, _) = SignedSecretKey::from_asc(
             "-----BEGIN PGP PRIVATE KEY BLOCK-----
 
 xcLYBF0fgz4BCADnRUV52V4xhSsU56ZaAn3+3oG86MZhXy4X8w14WZZDf0VJGeTh
@@ -577,58 +489,64 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 7yPJeQ==
 =KZk/
 -----END PGP PRIVATE KEY BLOCK-----",
-            KeyType::Private,
         )
-        .expect("failed to decode"); // NOTE: if you take out the ===GU1/ part, everything passes!
-        let binary = private_key.to_bytes();
-        Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        .expect("failed to decode");
+        let binary = DcKey::to_bytes(&private_key);
+        SignedSecretKey::from_slice(&binary).expect("invalid private key");
     }
 
     #[test]
-    fn test_format_fingerprint() {
-        let fingerprint = dc_format_fingerprint("1234567890ABCDABCDEFABCDEF1234567890ABCD");
+    fn test_asc_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedPublicKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
 
-        assert_eq!(
-            fingerprint,
-            "1234 5678 90AB CDAB CDEF\nABCD EF12 3456 7890 ABCD"
-        );
+        let key = KEYPAIR.secret.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedSecretKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
     }
 
     #[test]
     fn test_from_slice_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+        let public_key = KEYPAIR.public.clone();
+        let private_key = KEYPAIR.secret.clone();
 
-        let binary = public_key.to_bytes();
-        let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
+        let binary = DcKey::to_bytes(&public_key);
+        let public_key2 = SignedPublicKey::from_slice(&binary).expect("invalid public key");
         assert_eq!(public_key, public_key2);
 
-        let binary = private_key.to_bytes();
-        let private_key2 = Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        let binary = DcKey::to_bytes(&private_key);
+        let private_key2 = SignedSecretKey::from_slice(&binary).expect("invalid private key");
         assert_eq!(private_key, private_key2);
     }
 
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-
         for i in 0..4096 {
             bad_data[i] = (i & 0xff) as u8;
         }
-
         for j in 0..(4096 / 40) {
-            let bad_key = Key::from_slice(
-                &bad_data[j..j + 4096 / 2 + j],
-                if 0 != j & 1 {
-                    KeyType::Public
-                } else {
-                    KeyType::Private
-                },
-            );
-            assert!(bad_key.is_none());
+            let slice = &bad_data[j..j + 4096 / 2 + j];
+            assert!(SignedPublicKey::from_slice(slice).is_err());
+            assert!(SignedSecretKey::from_slice(slice).is_err());
         }
     }
 
+    #[test]
+    fn test_base64_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let base64 = key.to_base64();
+        let key2 = SignedPublicKey::from_base64(&base64).unwrap();
+        assert_eq!(key, key2);
+    }
+
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
@@ -641,7 +559,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_public() {
         let t = dummy_context().await;
         t.ctx
@@ -653,7 +570,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_secret() {
         let t = dummy_context().await;
         t.ctx
@@ -665,7 +581,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
@@ -686,29 +601,10 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(res0.unwrap(), res1.unwrap());
     }
 
-    #[test]
-    fn test_ascii_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
-
-        let s = public_key.to_armored_string(None).unwrap();
-        let (public_key2, _) =
-            Key::from_armored_string(&s, KeyType::Public).expect("invalid public key");
-        assert_eq!(public_key, public_key2);
-
-        let s = private_key.to_armored_string(None).unwrap();
-        println!("{}", &s);
-        let (private_key2, _) =
-            Key::from_armored_string(&s, KeyType::Private).expect("invalid private key");
-        assert_eq!(private_key, private_key2);
-    }
-
     #[test]
     fn test_split_key() {
-        let private_key = Key::from(KEYPAIR.secret.clone());
-        let public_wrapped = private_key.split_key().unwrap();
-        let public = SignedPublicKey::try_from(public_wrapped).unwrap();
-        assert_eq!(public.primary_key, KEYPAIR.public.primary_key);
+        let pubkey = KEYPAIR.secret.split_public_key().unwrap();
+        assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
     #[async_std::test]
@@ -756,4 +652,49 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     //     )
     //     .unwrap();
     // }
+
+    #[test]
+    fn test_fingerprint_from_str() {
+        let res = Fingerprint::new(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+
+        let fp: Fingerprint = "0102030405060708090A0B0c0d0e0F1011121314".parse().unwrap();
+        assert_eq!(fp, res);
+
+        let fp: Fingerprint = "zzzz 0102 0304 0506\n0708090a0b0c0D0E0F1011121314 yyy"
+            .parse()
+            .unwrap();
+        assert_eq!(fp, res);
+
+        let err = "1".parse::<Fingerprint>().err().unwrap();
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
+    }
+
+    #[test]
+    fn test_fingerprint_hex() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(fp.hex(), "0102040810204080FF0A0B0C0D0E0F1011121314");
+    }
+
+    #[test]
+    fn test_fingerprint_to_string() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(
+            fp.to_string(),
+            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF 1314"
+        );
+    }
 }

src/keyring.rs

@@ -1,46 +1,92 @@
-use std::borrow::Cow;
+//! Keyring to perform rpgp operations with.
+
+use anyhow::Result;
 
-use crate::constants::KeyType;
 use crate::context::Context;
-use crate::key::Key;
-use crate::sql::Sql;
+use crate::key::{self, DcKey};
 
-#[derive(Default, Clone, Debug)]
-pub struct Keyring<'a> {
-    keys: Vec<Cow<'a, Key>>,
+/// An in-memory keyring.
+///
+/// Instances are usually constructed just for the rpgp operation and
+/// short-lived.
+#[derive(Clone, Debug, Default)]
+pub struct Keyring<T>
+where
+    T: DcKey,
+{
+    keys: Vec<T>,
 }
 
-impl<'a> Keyring<'a> {
-    pub fn add_owned(&mut self, key: Key) {
-        self.add(Cow::Owned(key))
+impl<T> Keyring<T>
+where
+    T: DcKey<KeyType = T>,
+{
+    /// New empty keyring.
+    pub fn new() -> Keyring<T> {
+        Keyring { keys: Vec::new() }
     }
 
-    pub fn add_ref(&mut self, key: &'a Key) {
-        self.add(Cow::Borrowed(key))
+    /// Create a new keyring with the the user's secret key loaded.
+    pub async fn new_self(context: &Context) -> Result<Keyring<T>, key::Error> {
+        let mut keyring: Keyring<T> = Keyring::new();
+        keyring.load_self(context).await?;
+        Ok(keyring)
     }
 
-    fn add(&mut self, key: Cow<'a, Key>) {
+    /// Load the user's key into the keyring.
+    pub async fn load_self(&mut self, context: &Context) -> Result<(), key::Error> {
+        self.add(T::load_self(context).await?);
+        Ok(())
+    }
+
+    /// Add a key to the keyring.
+    pub fn add(&mut self, key: T) {
         self.keys.push(key);
     }
 
-    pub fn keys(&self) -> &[Cow<'a, Key>] {
-        &self.keys
+    pub fn len(&self) -> usize {
+        self.keys.len()
     }
 
-    pub async fn load_self_private_for_decrypting(
-        &mut self,
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> bool {
-        sql.query_get_value(
-            context,
-            "SELECT private_key FROM keypairs ORDER BY addr=? DESC, is_default DESC;",
-            paramsv![self_addr.as_ref().to_string()],
-        )
-        .await
-        .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Private))
-        .map(|key| self.add_owned(key))
-        .is_some()
+    pub fn is_empty(&self) -> bool {
+        self.keys.is_empty()
+    }
+
+    /// A vector with reference to all the keys in the keyring.
+    pub fn keys(&self) -> &[T] {
+        &self.keys
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::key::{SignedPublicKey, SignedSecretKey};
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_keyring_add_keys() {
+        let alice = alice_keypair();
+        let mut pub_ring: Keyring<SignedPublicKey> = Keyring::new();
+        pub_ring.add(alice.public.clone());
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let mut sec_ring: Keyring<SignedSecretKey> = Keyring::new();
+        sec_ring.add(alice.secret.clone());
+        assert_eq!(sec_ring.keys(), [alice.secret]);
+    }
+
+    #[async_std::test]
+    async fn test_keyring_load_self() {
+        // new_self() implies load_self()
+        let t = dummy_context().await;
+        configure_alice_keypair(&t.ctx).await;
+        let alice = alice_keypair();
+
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -11,8 +11,6 @@ extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 
@@ -69,6 +67,7 @@ mod simplify;
 mod smtp;
 pub mod stock;
 mod token;
+pub(crate) mod upload;
 #[macro_use]
 mod dehtml;
 

src/login_param.rs

@@ -130,10 +130,6 @@ impl LoginParam {
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
     pub async fn save_to_database(
         &self,
@@ -277,21 +273,15 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-    match certificate_checks {
-        CertificateChecks::Automatic => {
-            // Same as AcceptInvalidCertificates for now.
-            // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
-        }
-        CertificateChecks::Strict => tls_builder,
-        CertificateChecks::AcceptInvalidCertificates
-        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
+
+    if strict_tls {
+        tls_builder
+    } else {
+        tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
+            .danger_accept_invalid_certs(true)
     }
 }
 

src/lot.rs

@@ -1,5 +1,7 @@
 use deltachat_derive::{FromSql, ToSql};
 
+use crate::key::Fingerprint;
+
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
 /// Lot objects are created
@@ -14,7 +16,7 @@ pub struct Lot {
     pub(crate) timestamp: i64,
     pub(crate) state: LotState,
     pub(crate) id: u32,
-    pub(crate) fingerprint: Option<String>,
+    pub(crate) fingerprint: Option<Fingerprint>,
     pub(crate) invitenumber: Option<String>,
     pub(crate) auth: Option<String>,
 }

src/message.rs

@@ -1545,6 +1545,33 @@ pub async fn update_server_uid(
     }
 }
 
+/// Schedule attachement download for a message.
+pub async fn schedule_download(
+    context: &Context,
+    msg_id: MsgId,
+    path: Option<PathBuf>,
+) -> Result<(), Error> {
+    let msg = Message::load_from_db(context, msg_id).await?;
+    if let Some(_upload_url) = msg.param.get_upload_url() {
+        // TODO: Check if message was already downloaded.
+        let mut params = Params::new();
+        if let Some(path) = path {
+            params.set_upload_path(path);
+        }
+        job::add(
+            context,
+            job::Job::new(Action::DownloadMessageFile, msg_id.to_u32(), params, 0),
+        )
+        .await;
+    } else {
+        warn!(
+            context,
+            "Tried to schedule download for message {} which has no uploads", msg_id
+        );
+    }
+    Ok(())
+}
+
 #[allow(dead_code)]
 pub async fn dc_empty_server(context: &Context, flags: u32) {
     job::kill_action(context, Action::EmptyServer).await;

src/mimefactory.rs

@@ -50,6 +50,7 @@ pub struct MimeFactory<'a, 'b> {
     context: &'a Context,
     last_added_location_id: u32,
     attach_selfavatar: bool,
+    upload_url: Option<String>,
 }
 
 /// Result of rendering a message, ready to be submitted to a send job.
@@ -159,6 +160,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             last_added_location_id: 0,
             attach_selfavatar,
             context,
+            upload_url: None,
         };
         Ok(factory)
     }
@@ -206,6 +208,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             req_mdn: false,
             last_added_location_id: 0,
             attach_selfavatar: false,
+            upload_url: None,
         };
 
         Ok(res)
@@ -351,16 +354,47 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    let raw = message::get_summarytext_by_raw(
-                        self.msg.viewtype,
-                        self.msg.text.as_ref(),
-                        &self.msg.param,
-                        32,
-                        self.context,
-                    )
-                    .await;
-                    let raw_subject = raw.lines().next().unwrap_or_default();
-                    format!("Chat: {}", raw_subject)
+                    match chat.param.get(Param::LastSubject) {
+                        Some(last_subject) => {
+                            let subject_start = if last_subject.starts_with("Chat:") {
+                                0
+                            } else {
+                                // "Antw:" is the longest abbreviation in
+                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+                                // so look at the first _5_ characters:
+                                match last_subject.chars().take(5).position(|c| c == ':') {
+                                    Some(prefix_end) => prefix_end + 1,
+                                    None => 0,
+                                }
+                            };
+                            format!(
+                                "Re: {}",
+                                last_subject
+                                    .chars()
+                                    .skip(subject_start)
+                                    .collect::<String>()
+                                    .trim()
+                            )
+                        }
+                        None => {
+                            let self_name = match self.context.get_config(Config::Displayname).await
+                            {
+                                Some(name) => name,
+                                None => self
+                                    .context
+                                    .get_config(Config::Addr)
+                                    .await
+                                    .unwrap_or_default(),
+                            };
+
+                            self.context
+                                .stock_string_repl_str(
+                                    StockMessage::SubjectForNewContact,
+                                    self_name,
+                                )
+                                .await
+                        }
+                    }
                 }
             }
             Loaded::MDN { .. } => self
@@ -378,6 +412,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .collect()
     }
 
+    pub fn set_upload_url(&mut self, upload_url: String) {
+        self.upload_url = Some(upload_url)
+    }
+
     pub async fn render(mut self) -> Result<RenderedEmail, Error> {
         // Headers that are encrypted
         // - Chat-*, except Chat-Version
@@ -409,6 +447,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
+        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -463,7 +503,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let force_plaintext = self.should_force_plaintext();
         let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
-        let mut encrypt_helper = EncryptHelper::new(self.context).await?;
+        let encrypt_helper = EncryptHelper::new(self.context).await?;
 
         let subject = encode_words(&subject_str);
 
@@ -560,7 +600,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
 
             let encrypted = encrypt_helper
-                .encrypt(self.context, min_verified, message, &peerstates)
+                .encrypt(self.context, min_verified, message, peerstates)
                 .await?;
 
             outer_message = outer_message
@@ -846,11 +886,21 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        // if upload url is present: add as header and to message text
+        // TODO: make text part translatable (or remove)
+        let upload_url_text = if let Some(ref upload_url) = self.upload_url {
+            protected_headers.push(Header::new("Chat-Upload-Url".into(), upload_url.clone()));
+            Some(format!("\n\nFile attachement: {}", upload_url.clone()))
+        } else {
+            None
+        };
+
         let footer = &self.selfstatus;
         let message_text = format!(
-            "{}{}{}{}{}",
+            "{}{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
             escape_message_footer_marks(final_text),
+            upload_url_text.unwrap_or_default(),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -866,8 +916,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .body(message_text);
         let mut parts = Vec::new();
 
-        // add attachment part
-        if chat::msgtype_has_file(self.msg.viewtype) {
+        // add attachment part, skip if upload url was provided
+        if chat::msgtype_has_file(self.msg.viewtype) && self.upload_url.is_none() {
             if !is_file_size_okay(context, &self.msg).await {
                 bail!(
                     "Message exceeds the recommended {} MB.",
@@ -1174,6 +1224,11 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chatlist::Chatlist;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::mimeparser::*;
+    use crate::test_utils::configured_offline_context;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1181,6 +1236,9 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
+        assert!(!display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1192,6 +1250,25 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
+    #[test]
+    fn test_render_email_address_noescape() {
+        let display_name = "a space";
+        let addr = "x@y.org";
+
+        assert!(display_name.is_ascii());
+        assert!(display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+        let s = format!(
+            "{}",
+            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
+        );
+
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        assert_eq!(s, "a space <x@y.org>");
+    }
+
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1234,4 +1311,192 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
+
+    #[async_std::test]
+    async fn test_subject() {
+        // 1.: Receive a mail from an MUA or Delta Chat
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Infos: 42"
+        );
+
+        // 2. Receive a message from Delta Chat when we did not send any messages before
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Charlie <charlie@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        // 3. Send the first message to a new contact
+        let t = configured_offline_context().await;
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
+
+        let t = configured_offline_context().await;
+        t.ctx
+            .set_config(Config::Displayname, Some("Alice"))
+            .await
+            .unwrap();
+        assert_eq!(first_subject_str(t).await, "Message from Alice");
+
+        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.org>\n\
+            To: alice@example.org\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.org>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.org>\n\
+            To: alice@example.org\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.org>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+    }
+
+    async fn first_subject_str(t: TestContext) -> String {
+        let contact_id =
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.org", Origin::ManuallyCreated)
+                .await
+                .unwrap()
+                .0;
+
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+
+        mf.subject_str().await
+    }
+
+    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+        let t = configured_offline_context().await;
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        mf.subject_str().await
+    }
+
+    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+        context
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(context, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        new_msg
+    }
+
+    #[async_std::test]
+    // This test could still be extended
+    async fn test_render_reply() {
+        let t = configured_offline_context().await;
+        let context = &t.ctx;
+
+        let msg = incoming_msg_to_reply_msg(
+            b"From: Charlie <charlie@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            context,
+        )
+        .await;
+
+        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+
+        let recipients = mimefactory.recipients();
+        assert_eq!(recipients, vec!["charlie@example.org"]);
+
+        let rendered_msg = mimefactory.render().await.unwrap();
+
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        assert_eq!(
+            mail.headers
+                .iter()
+                .find(|h| h.get_key() == "MIME-Version")
+                .unwrap()
+                .get_value(),
+            "1.0"
+        );
+
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+            .await
+            .unwrap();
+    }
 }

src/mimeparser.rs

@@ -17,6 +17,7 @@ use crate::e2ee;
 use crate::error::{bail, Result};
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
 use crate::param::*;
@@ -44,7 +45,7 @@ pub struct MimeMessage {
     pub from: Vec<SingleInfo>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
-    pub signatures: HashSet<String>,
+    pub signatures: HashSet<Fingerprint>,
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
@@ -333,6 +334,13 @@ impl MimeMessage {
             }
         }
 
+        let upload_url = self.get(HeaderDef::ChatUploadUrl).map(|v| v.to_string());
+        if let Some(upload_url) = upload_url {
+            for part in self.parts.iter_mut() {
+                part.param.set_upload_url(upload_url.clone());
+            }
+        }
+
         self.parse_attachments();
 
         // See if an MDN is requested from the other side
@@ -526,6 +534,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
+                part.param.set(Param::Error, "Decryption failed");
 
                 self.parts.push(part);
 

src/param.rs

@@ -103,6 +103,9 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
+    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    LastSubject = b't',
+
     /// For Chats
     Devicetalk = b'D',
 
@@ -117,6 +120,12 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
+
+    /// For messages that have a HTTP file upload instead of attachement
+    UploadUrl = b'y',
+
+    /// For messages that have a HTTP file upload instead of attachement: Path to local file
+    UploadPath = b'Y',
 }
 
 /// Possible values for `Param::ForcePlaintext`.
@@ -314,6 +323,23 @@ impl Params {
         Ok(Some(path))
     }
 
+    pub fn get_upload_url(&self) -> Option<&str> {
+        self.get(Param::UploadUrl)
+    }
+
+    pub fn get_upload_path(&self, context: &Context) -> Result<Option<PathBuf>, BlobError> {
+        self.get_path(Param::UploadPath, context)
+    }
+
+    pub fn set_upload_path(&mut self, path: PathBuf) {
+        // TODO: Remove unwrap? May panic for invalid UTF8 in path.
+        self.set(Param::UploadPath, path.to_str().unwrap());
+    }
+
+    pub fn set_upload_url(&mut self, url: impl AsRef<str>) {
+        self.set(Param::UploadUrl, url);
+    }
+
     pub fn get_msg_id(&self) -> Option<MsgId> {
         self.get(Param::MsgId)
             .and_then(|x| x.parse::<u32>().ok())

src/peerstate.rs

@@ -6,9 +6,8 @@ use std::fmt;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
-use crate::constants::*;
 use crate::context::Context;
-use crate::key::{Key, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
 
 #[derive(Debug)]
@@ -32,13 +31,13 @@ pub struct Peerstate<'a> {
     pub last_seen: i64,
     pub last_seen_autocrypt: i64,
     pub prefer_encrypt: EncryptPreference,
-    pub public_key: Option<Key>,
-    pub public_key_fingerprint: Option<String>,
-    pub gossip_key: Option<Key>,
+    pub public_key: Option<SignedPublicKey>,
+    pub public_key_fingerprint: Option<Fingerprint>,
+    pub gossip_key: Option<SignedPublicKey>,
     pub gossip_timestamp: i64,
-    pub gossip_key_fingerprint: Option<String>,
-    pub verified_key: Option<Key>,
-    pub verified_key_fingerprint: Option<String>,
+    pub gossip_key_fingerprint: Option<Fingerprint>,
+    pub verified_key: Option<SignedPublicKey>,
+    pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
     pub degrade_event: Option<DegradeEvent>,
 }
@@ -127,7 +126,7 @@ impl<'a> Peerstate<'a> {
         res.last_seen_autocrypt = message_time;
         res.to_save = Some(ToSave::All);
         res.prefer_encrypt = header.prefer_encrypt;
-        res.public_key = Some(Key::from(header.public_key.clone()));
+        res.public_key = Some(header.public_key.clone());
         res.recalc_fingerprint();
 
         res
@@ -138,7 +137,7 @@ impl<'a> Peerstate<'a> {
 
         res.gossip_timestamp = message_time;
         res.to_save = Some(ToSave::All);
-        res.gossip_key = Some(Key::from(gossip_header.public_key.clone()));
+        res.gossip_key = Some(gossip_header.public_key.clone());
         res.recalc_fingerprint();
 
         res
@@ -152,7 +151,7 @@ impl<'a> Peerstate<'a> {
     pub async fn from_fingerprint(
         context: &'a Context,
         _sql: &Sql,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
     ) -> Option<Peerstate<'a>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
@@ -161,13 +160,8 @@ impl<'a> Peerstate<'a> {
                      WHERE public_key_fingerprint=? COLLATE NOCASE \
                      OR gossip_key_fingerprint=? COLLATE NOCASE  \
                      ORDER BY public_key_fingerprint=? DESC;";
-
-        Self::from_stmt(
-            context,
-            query,
-            paramsv![fingerprint, fingerprint, fingerprint],
-        )
-        .await
+        let fp = fingerprint.hex();
+        Self::from_stmt(context, query, paramsv![fp, fp, fp]).await
     }
 
     async fn from_stmt(
@@ -190,45 +184,30 @@ impl<'a> Peerstate<'a> {
                 res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
                 res.gossip_timestamp = row.get(5)?;
 
-                res.public_key_fingerprint = row.get(7)?;
-                if res
-                    .public_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.public_key_fingerprint = None;
-                }
-                res.gossip_key_fingerprint = row.get(8)?;
-                if res
-                    .gossip_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.gossip_key_fingerprint = None;
-                }
-                res.verified_key_fingerprint = row.get(10)?;
-                if res
-                    .verified_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.verified_key_fingerprint = None;
-                }
+                res.public_key_fingerprint = row
+                    .get::<_, Option<String>>(7)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.gossip_key_fingerprint = row
+                    .get::<_, Option<String>>(8)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.verified_key_fingerprint = row
+                    .get::<_, Option<String>>(10)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
                 res.public_key = row
                     .get(4)
                     .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
+                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
                 res.gossip_key = row
                     .get(6)
                     .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
+                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
                 res.verified_key = row
                     .get(9)
                     .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
+                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
 
                 Ok(res)
             })
@@ -300,8 +279,8 @@ impl<'a> Peerstate<'a> {
                 self.to_save = Some(ToSave::All)
             }
 
-            if self.public_key.as_ref() != Some(&Key::from(header.public_key.clone())) {
-                self.public_key = Some(Key::from(header.public_key.clone()));
+            if self.public_key.as_ref() != Some(&header.public_key) {
+                self.public_key = Some(header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All);
             }
@@ -316,9 +295,8 @@ impl<'a> Peerstate<'a> {
         if message_time > self.gossip_timestamp {
             self.gossip_timestamp = message_time;
             self.to_save = Some(ToSave::Timestamps);
-            let hdr_key = Key::from(gossip_header.public_key.clone());
-            if self.gossip_key.as_ref() != Some(&hdr_key) {
-                self.gossip_key = Some(hdr_key);
+            if self.gossip_key.as_ref() != Some(&gossip_header.public_key) {
+                self.gossip_key = Some(gossip_header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All)
             }
@@ -367,7 +345,16 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&Key> {
+    pub fn take_key(mut self, min_verified: PeerstateVerifiedStatus) -> Option<SignedPublicKey> {
+        match min_verified {
+            PeerstateVerifiedStatus::BidirectVerified => self.verified_key.take(),
+            PeerstateVerifiedStatus::Unverified => {
+                self.public_key.take().or_else(|| self.gossip_key.take())
+            }
+        }
+    }
+
+    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&SignedPublicKey> {
         match min_verified {
             PeerstateVerifiedStatus::BidirectVerified => self.verified_key.as_ref(),
             PeerstateVerifiedStatus::Unverified => self
@@ -380,7 +367,7 @@ impl<'a> Peerstate<'a> {
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
         verified: PeerstateVerifiedStatus,
     ) -> bool {
         if verified == PeerstateVerifiedStatus::BidirectVerified {
@@ -438,10 +425,10 @@ impl<'a> Peerstate<'a> {
                     self.public_key.as_ref().map(|k| k.to_bytes()),
                     self.gossip_timestamp,
                     self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    self.public_key_fingerprint,
-                    self.gossip_key_fingerprint,
+                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    self.verified_key_fingerprint,
+                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.addr,
                 ],
             ).await?;
@@ -462,7 +449,7 @@ impl<'a> Peerstate<'a> {
         Ok(())
     }
 
-    pub fn has_verified_key(&self, fingerprints: &HashSet<String>) -> bool {
+    pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
         if self.verified_key.is_some() && self.verified_key_fingerprint.is_some() {
             let vkc = self.verified_key_fingerprint.as_ref().unwrap();
             if fingerprints.contains(vkc) {
@@ -474,6 +461,12 @@ impl<'a> Peerstate<'a> {
     }
 }
 
+impl From<crate::key::FingerprintError> for rusqlite::Error {
+    fn from(_source: crate::key::FingerprintError) -> Self {
+        Self::InvalidColumnType(0, "Invalid fingerprint".into(), rusqlite::types::Type::Text)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -486,7 +479,7 @@ mod tests {
         let ctx = crate::test_utils::dummy_context().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
             context: &ctx.ctx,
@@ -528,7 +521,7 @@ mod tests {
     async fn test_peerstate_double_create() {
         let ctx = crate::test_utils::dummy_context().await;
         let addr = "hello@mail.com";
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let peerstate = Peerstate {
             context: &ctx.ctx,
@@ -562,7 +555,7 @@ mod tests {
         let ctx = crate::test_utils::dummy_context().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
             context: &ctx.ctx,

src/pgp.rs

@@ -1,7 +1,6 @@
 //! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
 
 use std::collections::{BTreeMap, HashSet};
-use std::convert::TryInto;
 use std::io;
 use std::io::Cursor;
 
@@ -11,6 +10,7 @@ use pgp::composed::{
     SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
+use pgp::ser::Serialize;
 use pgp::types::{
     CompressionAlgorithm, KeyTrait, Mpi, PublicKeyTrait, SecretKeyTrait, StringToKey,
 };
@@ -19,8 +19,8 @@ use rand::{thread_rng, CryptoRng, Rng};
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::key::*;
-use crate::keyring::*;
+use crate::key::{DcKey, Fingerprint};
+use crate::keyring::Keyring;
 
 pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
 pub const HEADER_SETUPCODE: &str = "passphrase-begin";
@@ -238,124 +238,152 @@ fn select_pk_for_encryption(key: &SignedPublicKey) -> Option<SignedPublicKeyOrSu
 
 /// Encrypts `plain` text using `public_keys_for_encryption`
 /// and signs it using `private_key_for_signing`.
-pub fn pk_encrypt(
+pub async fn pk_encrypt(
     plain: &[u8],
-    public_keys_for_encryption: &Keyring,
-    private_key_for_signing: Option<&Key>,
+    public_keys_for_encryption: Keyring<SignedPublicKey>,
+    private_key_for_signing: Option<SignedSecretKey>,
 ) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
-    let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            key.as_ref()
-                .try_into()
-                .ok()
-                .and_then(select_pk_for_encryption)
-        })
-        .collect();
-    let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    let mut rng = thread_rng();
+    async_std::task::spawn_blocking(move || {
+        let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
+            .keys()
+            .iter()
+            .filter_map(|key| select_pk_for_encryption(key))
+            .collect();
+        let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    // TODO: measure time
-    let encrypted_msg = if let Some(private_key) = private_key_for_signing {
-        let skey: &SignedSecretKey = private_key
-            .try_into()
-            .map_err(|_| format_err!("Invalid private key"))?;
+        let mut rng = thread_rng();
 
-        lit_msg
-            .sign(skey, || "".into(), Default::default())
-            .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
-            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
-    } else {
-        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
-    };
+        // TODO: measure time
+        let encrypted_msg = if let Some(ref skey) = private_key_for_signing {
+            lit_msg
+                .sign(skey, || "".into(), Default::default())
+                .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
+                .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
+        } else {
+            lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
+        };
 
-    let msg = encrypted_msg?;
-    let encoded_msg = msg.to_armored_string(None)?;
+        let msg = encrypted_msg?;
+        let encoded_msg = msg.to_armored_string(None)?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
+    .await
 }
 
 #[allow(clippy::implicit_hasher)]
-pub fn pk_decrypt(
-    ctext: &[u8],
-    private_keys_for_decryption: &Keyring,
-    public_keys_for_validation: &Keyring,
-    ret_signature_fingerprints: Option<&mut HashSet<String>>,
+pub async fn pk_decrypt(
+    ctext: Vec<u8>,
+    private_keys_for_decryption: Keyring<SignedSecretKey>,
+    public_keys_for_validation: Keyring<SignedPublicKey>,
+    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
 ) -> Result<Vec<u8>> {
-    let (msg, _) = Message::from_armor_single(Cursor::new(ctext))?;
-    let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            let k: &Key = &key;
-            k.try_into().ok()
-        })
-        .collect();
+    let msgs = async_std::task::spawn_blocking(move || {
+        let cursor = Cursor::new(ctext);
+        let (msg, _) = Message::from_armor_single(cursor)?;
+
+        let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption.keys().iter().collect();
+
+        let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
+        decryptor.collect::<pgp::errors::Result<Vec<_>>>()
+    })
+    .await?;
 
-    let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
     ensure!(!msgs.is_empty(), "No valid messages found");
 
-    let dec_msg = &msgs[0];
+    let content = match msgs[0].get_content()? {
+        Some(content) => content,
+        None => bail!("Decrypted message is empty"),
+    };
 
     if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-        if !public_keys_for_validation.keys().is_empty() {
-            let pkeys: Vec<&SignedPublicKey> = public_keys_for_validation
-                .keys()
-                .iter()
-                .filter_map(|key| {
-                    let k: &Key = &key;
-                    k.try_into().ok()
-                })
-                .collect();
+        if !public_keys_for_validation.is_empty() {
+            let fingerprints = async_std::task::spawn_blocking(move || {
+                let dec_msg = &msgs[0];
 
-            for pkey in &pkeys {
-                if dec_msg.verify(&pkey.primary_key).is_ok() {
-                    let fp = hex::encode_upper(pkey.fingerprint());
-                    ret_signature_fingerprints.insert(fp);
+                let pkeys = public_keys_for_validation.keys();
+
+                let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                for pkey in pkeys {
+                    if dec_msg.verify(&pkey.primary_key).is_ok() {
+                        let fp = DcKey::fingerprint(pkey);
+                        fingerprints.push(fp);
+                    }
                 }
-            }
+                fingerprints
+            })
+            .await;
+
+            ret_signature_fingerprints.extend(fingerprints);
         }
     }
 
-    match dec_msg.get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
-    }
+    Ok(content)
 }
 
-/// Symmetric encryption.
-pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
-    let mut rng = thread_rng();
-    let lit_msg = Message::new_literal_bytes("", plain);
-
-    let s2k = StringToKey::new_default(&mut rng);
-    let msg =
-        lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase.into())?;
-
-    let encoded_msg = msg.to_armored_string(None)?;
-
+/// Symmetric encryption with armored base64 text output.
+pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
+    let message = symm_encrypt_to_message(passphrase, plain).await?;
+    let encoded_msg = message.to_armored_string(None)?;
     Ok(encoded_msg)
 }
 
-/// Symmetric decryption.
-pub fn symm_decrypt<T: std::io::Read + std::io::Seek>(
+/// Symmetric encryption with binary output.
+pub async fn symm_encrypt_bytes(passphrase: &str, plain: &[u8]) -> Result<Vec<u8>> {
+    let message = symm_encrypt_to_message(passphrase, plain).await?;
+    let mut buf = Vec::new();
+    message.to_writer(&mut buf)?;
+    Ok(buf)
+}
+
+async fn symm_encrypt_to_message(passphrase: &str, plain: &[u8]) -> Result<Message> {
+    let lit_msg = Message::new_literal_bytes("", plain);
+    let passphrase = passphrase.to_string();
+
+    async_std::task::spawn_blocking(move || {
+        let mut rng = thread_rng();
+        let s2k = StringToKey::new_default(&mut rng);
+        let msg =
+            lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase)?;
+        Ok(msg)
+    })
+    .await
+}
+
+/// Symmetric decryption from armored text.
+pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     ctext: T,
 ) -> Result<Vec<u8>> {
     let (enc_msg, _) = Message::from_armor_single(ctext)?;
-    let decryptor = enc_msg.decrypt_with_password(|| passphrase.into())?;
+    symm_decrypt_from_message(enc_msg, passphrase).await
+}
 
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-    ensure!(!msgs.is_empty(), "No valid messages found");
+/// Symmetric decryption from bytes.
+pub async fn symm_decrypt_bytes<T: std::io::Read + std::io::Seek>(
+    passphrase: &str,
+    cbytes: T,
+) -> Result<Vec<u8>> {
+    let enc_msg = Message::from_bytes(cbytes)?;
+    symm_decrypt_from_message(enc_msg, passphrase).await
+}
 
-    match msgs[0].get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
-    }
+async fn symm_decrypt_from_message(message: Message, passphrase: &str) -> Result<Vec<u8>> {
+    let passphrase = passphrase.to_string();
+    async_std::task::spawn_blocking(move || {
+        let decryptor = message.decrypt_with_password(|| passphrase)?;
+
+        let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
+        ensure!(!msgs.is_empty(), "No valid messages found");
+
+        match msgs[0].get_content()? {
+            Some(content) => Ok(content),
+            None => bail!("Decrypted message is empty"),
+        }
+    })
+    .await
 }
 
 #[cfg(test)]
@@ -408,10 +436,10 @@ mod tests {
 
     /// [Key] objects to use in tests.
     struct TestKeys {
-        alice_secret: Key,
-        alice_public: Key,
-        bob_secret: Key,
-        bob_public: Key,
+        alice_secret: SignedSecretKey,
+        alice_public: SignedPublicKey,
+        bob_secret: SignedSecretKey,
+        bob_public: SignedPublicKey,
     }
 
     impl TestKeys {
@@ -419,10 +447,10 @@ mod tests {
             let alice = alice_keypair();
             let bob = bob_keypair();
             TestKeys {
-                alice_secret: Key::from(alice.secret.clone()),
-                alice_public: Key::from(alice.public.clone()),
-                bob_secret: Key::from(bob.secret.clone()),
-                bob_public: Key::from(bob.public.clone()),
+                alice_secret: alice.secret.clone(),
+                alice_public: alice.public.clone(),
+                bob_secret: bob.secret.clone(),
+                bob_public: bob.public.clone(),
             }
         }
     }
@@ -436,18 +464,18 @@ mod tests {
 
         /// A cyphertext encrypted to Alice & Bob, signed by Alice.
         static ref CTEXT_SIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, Some(&KEYS.alice_secret)).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
         };
 
         /// A cyphertext encrypted to Alice & Bob, not signed.
         static ref CTEXT_UNSIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, None).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
         };
     }
 
@@ -463,110 +491,115 @@ mod tests {
         assert!(CTEXT_UNSIGNED.starts_with("-----BEGIN PGP MESSAGE-----"));
     }
 
-    #[test]
-    fn test_decrypt_singed() {
+    #[async_std::test]
+    async fn test_decrypt_singed() {
         // Check decrypting as Alice
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring: Keyring<SignedSecretKey> = Keyring::new();
+        decrypt_keyring.add(KEYS.alice_secret.clone());
+        let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
 
         // Check decrypting as Bob
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
     }
 
-    #[test]
-    fn test_decrypt_no_sig_check() {
-        let mut keyring = Keyring::default();
-        keyring.add_ref(&KEYS.alice_secret);
-        let empty_keyring = Keyring::default();
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_no_sig_check() {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_secret.clone());
+        let empty_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &keyring,
-            &empty_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            keyring,
+            empty_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_key() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_key() {
         // The validation does not have the public key of the signer.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.bob_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.bob_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_unsigned() {
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let sig_check_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_unsigned() {
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let sig_check_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_UNSIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_UNSIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_sigret() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_sigret() {
         // Check decrypting signed cyphertext without providing the HashSet for signatures.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             None,
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
     }

src/provider/data.rs

@@ -19,6 +19,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // aol.md: aol.com
@@ -30,6 +31,7 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // autistici.org.md: autistici.org
@@ -43,6 +45,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -56,6 +59,21 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+    };
+
+    // chello.at.md: chello.at
+    static ref P_CHELLO_AT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/chello-at",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "mail.mymagenta.at", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.mymagenta.at", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
     };
 
     // comcast.md: xfinity.com, comcast.net
@@ -65,7 +83,16 @@ lazy_static::lazy_static! {
     // - skipping provider with status OK and no special things to do
 
     // disroot.md: disroot.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISROOT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/disroot",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // example.com.md: example.com, example.org
     static ref P_EXAMPLE_COM: Provider = Provider {
@@ -78,6 +105,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // fastmail.md: fastmail.com
@@ -89,6 +117,19 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+    };
+
+    // five.chat.md: five.chat
+    static ref P_FIVE_CHAT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/five-chat",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
     };
 
     // freenet.de.md: freenet.de
@@ -102,6 +143,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -115,6 +157,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -129,6 +172,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // i.ua.md: i.ua
@@ -145,6 +189,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // kolst.com.md: kolst.com
@@ -157,7 +202,16 @@ lazy_static::lazy_static! {
     // - skipping provider with status OK and no special things to do
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAILBOX_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mailbox-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // nauta.cu.md: nauta.cu
     static ref P_NAUTA_CU: Provider = Provider {
@@ -178,6 +232,7 @@ lazy_static::lazy_static! {
             ConfigDefault { key: Config::E2eeEnabled, value: "0" },
             ConfigDefault { key: Config::MediaQuality, value: "1" },
         ]),
+        strict_tls: false,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -191,9 +246,10 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
-    // posteo.md: posteo.de
+    // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
     static ref P_POSTEO: Provider = Provider {
         status: Status::OK,
         before_login_hint: "",
@@ -204,6 +260,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -215,16 +272,35 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // riseup.net.md: riseup.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_RISEUP_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/riseup-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // rogers.com.md: rogers.com
     // - skipping provider with status OK and no special things to do
 
     // systemli.org.md: systemli.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_SYSTEMLI_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/systemli-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
@@ -235,6 +311,7 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // testrun.md: testrun.org
@@ -249,6 +326,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // tiscali.it.md: tiscali.it
@@ -262,6 +340,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // ukr.net.md: ukr.net
@@ -282,12 +361,13 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
     static ref P_YAHOO: Provider = Provider {
         status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with your Yahoo email address you have to allow \"less secure apps\" in the Yahoo webinterface.",
+        before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yahoo",
         server: vec![
@@ -295,6 +375,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // yandex.ru.md: yandex.ru, yandex.com
@@ -306,6 +387,7 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -319,6 +401,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
@@ -326,9 +409,12 @@ lazy_static::lazy_static! {
         ("aol.com", &*P_AOL),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("chello.at", &*P_CHELLO_AT),
+        ("disroot.org", &*P_DISROOT),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -344,6 +430,8 @@ lazy_static::lazy_static! {
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
@@ -351,8 +439,58 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
+        ("posteo.af", &*P_POSTEO),
+        ("posteo.at", &*P_POSTEO),
+        ("posteo.be", &*P_POSTEO),
+        ("posteo.ch", &*P_POSTEO),
+        ("posteo.cl", &*P_POSTEO),
+        ("posteo.co", &*P_POSTEO),
+        ("posteo.co.uk", &*P_POSTEO),
+        ("posteo.com.br", &*P_POSTEO),
+        ("posteo.cr", &*P_POSTEO),
+        ("posteo.cz", &*P_POSTEO),
+        ("posteo.dk", &*P_POSTEO),
+        ("posteo.ee", &*P_POSTEO),
+        ("posteo.es", &*P_POSTEO),
+        ("posteo.eu", &*P_POSTEO),
+        ("posteo.fi", &*P_POSTEO),
+        ("posteo.gl", &*P_POSTEO),
+        ("posteo.gr", &*P_POSTEO),
+        ("posteo.hn", &*P_POSTEO),
+        ("posteo.hr", &*P_POSTEO),
+        ("posteo.hu", &*P_POSTEO),
+        ("posteo.ie", &*P_POSTEO),
+        ("posteo.in", &*P_POSTEO),
+        ("posteo.is", &*P_POSTEO),
+        ("posteo.jp", &*P_POSTEO),
+        ("posteo.la", &*P_POSTEO),
+        ("posteo.li", &*P_POSTEO),
+        ("posteo.lt", &*P_POSTEO),
+        ("posteo.lu", &*P_POSTEO),
+        ("posteo.me", &*P_POSTEO),
+        ("posteo.mx", &*P_POSTEO),
+        ("posteo.my", &*P_POSTEO),
+        ("posteo.net", &*P_POSTEO),
+        ("posteo.nl", &*P_POSTEO),
+        ("posteo.no", &*P_POSTEO),
+        ("posteo.nz", &*P_POSTEO),
+        ("posteo.org", &*P_POSTEO),
+        ("posteo.pe", &*P_POSTEO),
+        ("posteo.pl", &*P_POSTEO),
+        ("posteo.pm", &*P_POSTEO),
+        ("posteo.pt", &*P_POSTEO),
+        ("posteo.ro", &*P_POSTEO),
+        ("posteo.ru", &*P_POSTEO),
+        ("posteo.se", &*P_POSTEO),
+        ("posteo.sg", &*P_POSTEO),
+        ("posteo.si", &*P_POSTEO),
+        ("posteo.tn", &*P_POSTEO),
+        ("posteo.uk", &*P_POSTEO),
+        ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),

src/provider/mod.rs

@@ -72,6 +72,7 @@ pub struct Provider {
     pub overview_page: &'static str,
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
+    pub strict_tls: bool,
 }
 
 impl Provider {

src/provider/update.py

@@ -100,6 +100,9 @@ def process_data(data, file):
 
     config_defaults = process_config_defaults(data)
 
+    strict_tls = data.get("strict_tls", False)
+    strict_tls = "true" if strict_tls else "false"
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -111,6 +114,7 @@ def process_data(data, file):
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
         provider += "        config_defaults: " + config_defaults + ",\n"
+        provider += "        strict_tls: " + strict_tls + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -121,7 +125,7 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
+    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None" and strict_tls == "false":
         out_all += "    // - skipping provider with status OK and no special things to do\n\n"
     else:
         out_all += provider

src/qr.rs

@@ -10,8 +10,7 @@ use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::key::dc_format_fingerprint;
-use crate::key::dc_normalize_fingerprint;
+use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
@@ -80,6 +79,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         Some(pair) => pair,
         None => (payload, ""),
     };
+    let fingerprint: Fingerprint = match fingerprint.parse() {
+        Ok(fp) => fp,
+        Err(err) => {
+            return Error::new(err)
+                .context("Failed to parse fingerprint in QR code")
+                .into()
+        }
+    };
 
     // replace & with \n to match expected param format
     let fragment = fragment.replace('&', "\n");
@@ -128,13 +135,6 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         None
     };
 
-    let fingerprint = dc_normalize_fingerprint(fingerprint);
-
-    // ensure valid fingerprint
-    if fingerprint.len() != 40 {
-        return format_err!("Bad fingerprint length in QR code").into();
-    }
-
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
@@ -161,7 +161,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr)).await;
         } else {
             lot.state = LotState::QrFprWithoutAddr;
-            lot.text1 = Some(dc_format_fingerprint(&fingerprint));
+            lot.text1 = Some(fingerprint.to_string());
         }
     } else if let Some(addr) = addr {
         if grpid.is_some() && grpname.is_some() {

src/scheduler.rs

@@ -2,12 +2,10 @@ use async_std::prelude::*;
 use async_std::sync::{channel, Receiver, Sender};
 use async_std::task;
 
-use std::time::Duration;
-
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::smtp::Smtp;
+use crate::{config::Config, message::MsgId, smtp::Smtp};
 
 pub(crate) struct StopToken;
 
@@ -25,30 +23,29 @@ pub(crate) enum Scheduler {
         sentbox_handle: Option<task::JoinHandle<()>>,
         smtp: SmtpConnectionState,
         smtp_handle: Option<task::JoinHandle<()>>,
-        probe_network: bool,
     },
 }
 
 impl Context {
     /// Indicate that the network likely has come back.
     pub async fn maybe_network(&self) {
-        self.scheduler.write().await.maybe_network().await;
+        self.scheduler.read().await.maybe_network().await;
     }
 
-    pub(crate) async fn interrupt_inbox(&self) {
-        self.scheduler.read().await.interrupt_inbox().await;
+    pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_inbox(info).await;
     }
 
-    pub(crate) async fn interrupt_sentbox(&self) {
-        self.scheduler.read().await.interrupt_sentbox().await;
+    pub(crate) async fn interrupt_sentbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_sentbox(info).await;
     }
 
-    pub(crate) async fn interrupt_mvbox(&self) {
-        self.scheduler.read().await.interrupt_mvbox().await;
+    pub(crate) async fn interrupt_mvbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_mvbox(info).await;
     }
 
-    pub(crate) async fn interrupt_smtp(&self) {
-        self.scheduler.read().await.interrupt_smtp().await;
+    pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_smtp(info).await;
     }
 }
 
@@ -66,33 +63,26 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         // track number of continously executed jobs
         let mut jobs_loaded = 0;
+        let mut info = InterruptInfo::default();
         loop {
-            let probe_network = ctx.scheduler.read().await.get_probe_network();
-            match job::load_next(&ctx, Thread::Imap, probe_network)
-                .timeout(Duration::from_millis(200))
-                .await
-            {
-                Ok(Some(job)) if jobs_loaded <= 20 => {
+            match job::load_next(&ctx, Thread::Imap, &info).await {
+                Some(job) if jobs_loaded <= 20 => {
                     jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
-                    ctx.scheduler.write().await.set_probe_network(false);
+                    info = Default::default();
                 }
-                Ok(Some(job)) => {
+                Some(job) => {
                     // Let the fetch run, but return back to the job afterwards.
                     info!(ctx, "postponing imap-job {} to run fetch...", job);
                     jobs_loaded = 0;
                     fetch(&ctx, &mut connection).await;
                 }
-                Ok(None) | Err(async_std::future::TimeoutError { .. }) => {
+                None => {
                     jobs_loaded = 0;
-                    fetch_idle(&ctx, &mut connection).await;
+                    info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
                 }
             }
         }
@@ -109,15 +99,18 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 }
 
 async fn fetch(ctx: &Context, connection: &mut Imap) {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "{}", err);
+                return;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
         }
         None => {
             warn!(ctx, "Can not fetch inbox folder, not set");
@@ -126,16 +119,20 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     }
 }
 
-async fn fetch_idle(ctx: &Context, connection: &mut Imap) {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
+    match ctx.get_config(folder).await {
         Some(watch_folder) => {
+            // connect and fake idle if unable to connect
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "imap connection failed: {}", err);
+                return connection.fake_idle(&ctx, None).await;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
 
             // idle
             if connection.can_idle() {
@@ -143,15 +140,17 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap) {
                     .idle(&ctx, Some(watch_folder))
                     .await
                     .unwrap_or_else(|err| {
-                        error!(ctx, "{}", err);
-                    });
+                        connection.trigger_reconnect();
+                        warn!(ctx, "{}", err);
+                        InterruptInfo::new(false, None)
+                    })
             } else {
-                connection.fake_idle(&ctx, Some(watch_folder)).await;
+                connection.fake_idle(&ctx, Some(watch_folder)).await
             }
         }
         None => {
-            warn!(ctx, "Can not watch inbox folder, not set");
-            connection.fake_idle(&ctx, None).await;
+            warn!(ctx, "Can not watch {} folder, not set", folder);
+            connection.fake_idle(&ctx, None).await
         }
     }
 }
@@ -160,7 +159,7 @@ async fn simple_imap_loop(
     ctx: Context,
     started: Sender<()>,
     inbox_handlers: ImapConnectionHandlers,
-    folder: impl AsRef<str>,
+    folder: Config,
 ) {
     use futures::future::FutureExt;
 
@@ -176,43 +175,9 @@ async fn simple_imap_loop(
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         loop {
-            match get_watch_folder(&ctx, folder.as_ref()).await {
-                Some(watch_folder) => {
-                    // fetch
-                    connection
-                        .fetch(&ctx, &watch_folder)
-                        .await
-                        .unwrap_or_else(|err| {
-                            error!(ctx, "{}", err);
-                        });
-
-                    // idle
-                    if connection.can_idle() {
-                        connection
-                            .idle(&ctx, Some(watch_folder))
-                            .await
-                            .unwrap_or_else(|err| {
-                                error!(ctx, "{}", err);
-                            });
-                    } else {
-                        connection.fake_idle(&ctx, Some(watch_folder)).await;
-                    }
-                }
-                None => {
-                    warn!(
-                        &ctx,
-                        "No watch folder found for {}, skipping",
-                        folder.as_ref()
-                    );
-                    connection.fake_idle(&ctx, None).await
-                }
-            }
+            fetch_idle(&ctx, &mut connection, folder).await;
         }
     };
 
@@ -241,25 +206,20 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
+
+        let mut interrupt_info = Default::default();
         loop {
-            let probe_network = ctx.scheduler.read().await.get_probe_network();
-            match job::load_next(&ctx, Thread::Smtp, probe_network)
-                .timeout(Duration::from_millis(200))
-                .await
-            {
-                Ok(Some(job)) => {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
+                Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
-                    ctx.scheduler.write().await.set_probe_network(false);
+                    interrupt_info = Default::default();
                 }
-                Ok(None) | Err(async_std::future::TimeoutError { .. }) => {
-                    info!(ctx, "smtp fake idle");
+                None => {
                     // Fake Idle
-                    idle_interrupt_receiver
-                        .recv()
-                        .timeout(Duration::from_secs(5))
-                        .await
-                        .ok();
+                    info!(ctx, "smtp fake idle - started");
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    info!(ctx, "smtp fake idle - interrupted")
                 }
             }
         }
@@ -288,7 +248,6 @@ impl Scheduler {
             mvbox,
             sentbox,
             smtp,
-            probe_network: false,
             inbox_handle: None,
             mvbox_handle: None,
             sentbox_handle: None,
@@ -311,7 +270,7 @@ impl Scheduler {
                     ctx1,
                     mvbox_start_send,
                     mvbox_handlers,
-                    "configured_mvbox_folder",
+                    Config::ConfiguredMvboxFolder,
                 )
                 .await
             }));
@@ -325,7 +284,7 @@ impl Scheduler {
                     ctx1,
                     sentbox_start_send,
                     sentbox_handlers,
-                    "configured_sentbox_folder",
+                    Config::ConfiguredSentboxFolder,
                 )
                 .await
             }));
@@ -353,58 +312,39 @@ impl Scheduler {
         info!(ctx, "scheduler is running");
     }
 
-    fn set_probe_network(&mut self, val: bool) {
-        match self {
-            Scheduler::Running {
-                ref mut probe_network,
-                ..
-            } => {
-                *probe_network = val;
-            }
-            _ => panic!("set_probe_network can only be called when running"),
-        }
-    }
-
-    fn get_probe_network(&self) -> bool {
-        match self {
-            Scheduler::Running { probe_network, .. } => *probe_network,
-            _ => panic!("get_probe_network can only be called when running"),
-        }
-    }
-
-    async fn maybe_network(&mut self) {
+    async fn maybe_network(&self) {
         if !self.is_running() {
             return;
         }
-        self.set_probe_network(true);
-        self.interrupt_inbox()
-            .join(self.interrupt_mvbox())
-            .join(self.interrupt_sentbox())
-            .join(self.interrupt_smtp())
+
+        self.interrupt_inbox(InterruptInfo::new(true, None))
+            .join(self.interrupt_mvbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_smtp(InterruptInfo::new(true, None)))
             .await;
     }
 
-    async fn interrupt_inbox(&self) {
+    async fn interrupt_inbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref inbox, .. } = self {
-            inbox.interrupt().await;
+            inbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_mvbox(&self) {
+    async fn interrupt_mvbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref mvbox, .. } = self {
-            mvbox.interrupt().await;
+            mvbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_sentbox(&self) {
+    async fn interrupt_sentbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref sentbox, .. } = self {
-            sentbox.interrupt().await;
+            sentbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_smtp(&self) {
+    async fn interrupt_smtp(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref smtp, .. } = self {
-            smtp.interrupt().await;
+            smtp.interrupt(info).await;
         }
     }
 
@@ -473,7 +413,7 @@ struct ConnectionState {
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
-    idle_interrupt_sender: Sender<()>,
+    idle_interrupt_sender: Sender<InterruptInfo>,
 }
 
 impl ConnectionState {
@@ -485,11 +425,9 @@ impl ConnectionState {
         self.shutdown_receiver.recv().await.ok();
     }
 
-    async fn interrupt(&self) {
-        if !self.idle_interrupt_sender.is_full() {
-            // Use try_send to avoid blocking on interrupts.
-            self.idle_interrupt_sender.send(()).await;
-        }
+    async fn interrupt(&self, info: InterruptInfo) {
+        // Use try_send to avoid blocking on interrupts.
+        self.idle_interrupt_sender.try_send(info).ok();
     }
 }
 
@@ -523,8 +461,8 @@ impl SmtpConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self) {
-        self.state.interrupt().await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -533,12 +471,11 @@ impl SmtpConnectionState {
     }
 }
 
-#[derive(Debug)]
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
     shutdown_sender: Sender<()>,
-    idle_interrupt_receiver: Receiver<()>,
+    idle_interrupt_receiver: Receiver<InterruptInfo>,
 }
 
 #[derive(Debug)]
@@ -550,8 +487,8 @@ impl ImapConnectionState {
     /// Construct a new connection.
     fn new() -> (Self, ImapConnectionHandlers) {
         let (stop_sender, stop_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
         let (shutdown_sender, shutdown_receiver) = channel(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new(idle_interrupt_receiver),
@@ -571,8 +508,8 @@ impl ImapConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self) {
-        self.state.interrupt().await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -588,20 +525,17 @@ struct ImapConnectionHandlers {
     shutdown_sender: Sender<()>,
 }
 
-async fn get_watch_folder(context: &Context, config_name: impl AsRef<str>) -> Option<String> {
-    match context
-        .sql
-        .get_raw_config(context, config_name.as_ref())
-        .await
-    {
-        Some(name) => Some(name),
-        None => {
-            if config_name.as_ref() == "configured_inbox_folder" {
-                // initialized with old version, so has not set configured_inbox_folder
-                Some("INBOX".to_string())
-            } else {
-                None
-            }
+#[derive(Default, Debug)]
+pub struct InterruptInfo {
+    pub probe_network: bool,
+    pub msg_id: Option<MsgId>,
+}
+
+impl InterruptInfo {
+    pub fn new(probe_network: bool, msg_id: Option<MsgId>) -> Self {
+        Self {
+            probe_network,
+            msg_id,
         }
     }
 }

src/securejoin.rs

@@ -14,7 +14,7 @@ use crate::e2ee::*;
 use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, DcKey, Key, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -73,8 +73,6 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     ====   Step 1 in "Setup verified contact" protocol   ====
     =======================================================*/
 
-    let fingerprint: String;
-
     ensure_secret_key_exists(context).await.ok();
 
     // invitenumber will be used to allow starting the handshake,
@@ -95,7 +93,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         .await
         .unwrap_or_default();
 
-    fingerprint = match get_self_fingerprint(context).await {
+    let fingerprint: Fingerprint = match get_self_fingerprint(context).await {
         Some(fp) => fp,
         None => {
             return None;
@@ -140,9 +138,9 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     qr
 }
 
-async fn get_self_fingerprint(context: &Context) -> Option<String> {
+async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
     match SignedPublicKey::load_self(context).await {
-        Ok(key) => Some(Key::from(key).fingerprint()),
+        Ok(key) => Some(key.fingerprint()),
         Err(_) => {
             warn!(context, "get_self_fingerprint(): failed to load key");
             None
@@ -249,7 +247,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
             chat_id_2_contact_id(context, contact_chat_id).await,
             400
         );
-        let own_fingerprint = get_self_fingerprint(context).await.unwrap_or_default();
+        let own_fingerprint = get_self_fingerprint(context).await;
 
         // Bob -> Alice
         if let Err(err) = send_handshake_msg(
@@ -261,7 +259,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
                 "vc-request-with-auth"
             },
             get_qr_attr!(context, auth).to_string(),
-            Some(own_fingerprint),
+            own_fingerprint,
             if join_vg {
                 get_qr_attr!(context, text2).to_string()
             } else {
@@ -311,7 +309,7 @@ async fn send_handshake_msg(
     contact_chat_id: ChatId,
     step: &str,
     param2: impl AsRef<str>,
-    fingerprint: Option<String>,
+    fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
 ) -> Result<(), HandshakeError> {
     let mut msg = Message::default();
@@ -328,7 +326,7 @@ async fn send_handshake_msg(
         msg.param.set(Param::Arg2, param2);
     }
     if let Some(fp) = fingerprint {
-        msg.param.set(Param::Arg3, fp);
+        msg.param.set(Param::Arg3, fp.hex());
     }
     if !grpid.as_ref().is_empty() {
         msg.param.set(Param::Arg4, grpid.as_ref());
@@ -360,7 +358,7 @@ async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32
 
 async fn fingerprint_equals_sender(
     context: &Context,
-    fingerprint: impl AsRef<str>,
+    fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
 ) -> bool {
     let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
@@ -368,9 +366,8 @@ async fn fingerprint_equals_sender(
     if contacts.len() == 1 {
         if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
             if let Some(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
-                let fingerprint_normalized = dc_normalize_fingerprint(fingerprint.as_ref());
                 if peerstate.public_key_fingerprint.is_some()
-                    && &fingerprint_normalized == peerstate.public_key_fingerprint.as_ref().unwrap()
+                    && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
                     return true;
                 }
@@ -397,6 +394,8 @@ pub(crate) enum HandshakeError {
     NoSelfAddr,
     #[error("Failed to send message")]
     MsgSendFailed(#[source] Error),
+    #[error("Failed to parse fingerprint")]
+    BadFingerprint(#[from] crate::key::FingerprintError),
 }
 
 /// What to do with a Secure-Join handshake message after it was handled.
@@ -516,10 +515,11 @@ pub(crate) async fn handle_securejoin_handshake(
                 // no error, just aborted somehow or a mail from another handshake
                 return Ok(HandshakeMessage::Ignore);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
             let auth = get_qr_attr!(context, auth).to_string();
 
-            if !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice) {
+            if !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -576,8 +576,9 @@ pub(crate) async fn handle_securejoin_handshake(
             ==========================================================*/
 
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -588,7 +589,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !encrypted_and_signed(context, mime_message, &fingerprint) {
+            if !encrypted_and_signed(context, mime_message, Some(&fingerprint)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -625,7 +626,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -673,7 +674,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     contact_chat_id,
                     "vc-contact-confirm",
                     "",
-                    Some(fingerprint.clone()),
+                    Some(fingerprint),
                     "",
                 )
                 .await?;
@@ -709,7 +710,8 @@ pub(crate) async fn handle_securejoin_handshake(
                 );
                 return Ok(abort_retval);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
@@ -731,7 +733,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 true
             };
             if vg_expect_encrypted
-                && !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice)
+                && !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice))
             {
                 could_not_establish_secure_connection(
                     context,
@@ -888,7 +890,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
             if !encrypted_and_signed(
                 context,
                 mime_message,
-                get_self_fingerprint(context).await.unwrap_or_default(),
+                get_self_fingerprint(context).await.as_ref(),
             ) {
                 could_not_establish_secure_connection(
                     context,
@@ -898,8 +900,9 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -910,7 +913,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -967,16 +970,13 @@ async fn could_not_establish_secure_connection(
     error!(context, "{} ({})", &msg, details);
 }
 
-async fn mark_peer_as_verified(
-    context: &Context,
-    fingerprint: impl AsRef<str>,
-) -> Result<(), Error> {
+async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
     if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint.as_ref()).await
+        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await
     {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
-            fingerprint.as_ref(),
+            fingerprint,
             PeerstateVerifiedStatus::BidirectVerified,
         ) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
@@ -990,7 +990,7 @@ async fn mark_peer_as_verified(
     }
     bail!(
         "could not mark peer as verified for fingerprint {}",
-        fingerprint.as_ref()
+        fingerprint.hex()
     );
 }
 
@@ -1001,7 +1001,7 @@ async fn mark_peer_as_verified(
 fn encrypted_and_signed(
     context: &Context,
     mimeparser: &MimeMessage,
-    expected_fingerprint: impl AsRef<str>,
+    expected_fingerprint: Option<&Fingerprint>,
 ) -> bool {
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
@@ -1009,17 +1009,17 @@ fn encrypted_and_signed(
     } else if mimeparser.signatures.is_empty() {
         warn!(context, "Message not signed.",);
         false
-    } else if expected_fingerprint.as_ref().is_empty() {
-        warn!(context, "Fingerprint for comparison missing.",);
+    } else if expected_fingerprint.is_none() {
+        warn!(context, "Fingerprint for comparison missing.");
         false
     } else if !mimeparser
         .signatures
-        .contains(expected_fingerprint.as_ref())
+        .contains(expected_fingerprint.unwrap())
     {
         warn!(
             context,
             "Message does not match expected fingerprint {}.",
-            expected_fingerprint.as_ref(),
+            expected_fingerprint.unwrap(),
         );
         false
     } else {

src/smtp/mod.rs

@@ -10,8 +10,9 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
-use crate::login_param::{dc_build_tls, LoginParam};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::oauth2::*;
+use crate::provider::get_provider_info;
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -44,9 +45,8 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Default, DebugStub)]
-pub struct Smtp {
-    #[debug_stub(some = "SmtpTransport")]
+#[derive(Default)]
+pub(crate) struct Smtp {
     transport: Option<smtp::SmtpTransport>,
 
     /// Email address we are sending from.
@@ -113,7 +113,14 @@ impl Smtp {
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls_config = dc_build_tls(lp.smtp_certificate_checks);
+        let provider = get_provider_info(&lp.addr);
+        let strict_tls = match lp.smtp_certificate_checks {
+            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Strict => true,
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => false,
+        };
+        let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
@@ -172,7 +179,7 @@ impl Smtp {
                 .stock_string_repl_str2(
                     StockMessage::ServerResponse,
                     format!("SMTP {}:{}", domain, port),
-                    err.to_string(),
+                    format!("{}, ({:?})", err.to_string(), err),
                 )
                 .await;
 

src/sql.rs

@@ -49,7 +49,7 @@ pub enum Error {
 pub type Result<T> = std::result::Result<T, Error>;
 
 /// A wrapper around the underlying Sqlite3 object.
-#[derive(DebugStub)]
+#[derive(Debug)]
 pub struct Sql {
     pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
 }

src/stock.rs

@@ -179,6 +179,9 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
     UnknownSenderForChat = 72,
+
+    #[strum(props(fallback = "Message from %1$s"))]
+    SubjectForNewContact = 73,
 }
 
 /*

src/test_utils.rs

@@ -40,6 +40,23 @@ pub(crate) async fn dummy_context() -> TestContext {
     test_context().await
 }
 
+pub(crate) async fn configured_offline_context() -> TestContext {
+    let t = dummy_context().await;
+    t.ctx
+        .set_config(Config::Addr, Some("alice@example.org"))
+        .await
+        .unwrap();
+    t.ctx
+        .set_config(Config::ConfiguredAddr, Some("alice@example.org"))
+        .await
+        .unwrap();
+    t.ctx
+        .set_config(Config::Configured, Some("1"))
+        .await
+        .unwrap();
+    t
+}
+
 /// Load a pre-generated keypair for alice@example.com from disk.
 ///
 /// This saves CPU cycles by avoiding having to generate a key.

src/upload.rs

@@ -0,0 +1,142 @@
+use crate::blob::BlobObject;
+// use crate::constants::Viewtype;
+use crate::context::Context;
+use crate::error::{bail, format_err, Result};
+use crate::message::{Message, MsgId};
+use crate::pgp::{symm_decrypt_bytes, symm_encrypt_bytes};
+use async_std::fs;
+use async_std::path::PathBuf;
+use rand::Rng;
+use std::io::Cursor;
+use url::Url;
+
+/// Upload file to a HTTP upload endpoint.
+pub async fn upload_file(
+    context: &Context,
+    url: impl AsRef<str>,
+    filepath: PathBuf,
+) -> Result<String> {
+    let (passphrase, url) = parse_upload_url(url)?;
+
+    let content = fs::read(filepath).await?;
+    let encrypted = symm_encrypt_bytes(&passphrase, &content).await?;
+
+    // TODO: Use tokens for upload.
+    info!(context, "uploading encrypted file to {}", &url);
+    let response = surf::put(url).body_bytes(encrypted).await;
+    if let Err(err) = response {
+        bail!("Upload failed: {}", err);
+    }
+    let mut response = response.unwrap();
+    match response.body_string().await {
+        Ok(string) => Ok(string),
+        Err(err) => bail!("Invalid response from upload: {}", err),
+    }
+}
+
+pub async fn download_message_file(
+    context: &Context,
+    msg_id: MsgId,
+    download_path: Option<PathBuf>,
+) -> Result<()> {
+    let mut message = Message::load_from_db(context, msg_id).await?;
+    let upload_url = message
+        .param
+        .get_upload_url()
+        .ok_or_else(|| format_err!("Message has no upload URL"))?;
+
+    let (passphrase, url) = parse_upload_url(upload_url)?;
+
+    let filename: String = url
+        .path_segments()
+        .ok_or_else(|| format_err!("Invalid upload URL"))?
+        .last()
+        .ok_or_else(|| format_err!("Invalid upload URL"))?
+        .to_string();
+
+    let data = download_file(context, url, passphrase).await?;
+    let saved_path = if let Some(download_path) = download_path {
+        fs::write(&download_path, data).await?;
+        download_path.to_string_lossy().to_string()
+    } else {
+        let blob = BlobObject::create(context, filename.clone(), &data)
+            .await
+            .map_err(|err| {
+                format_err!(
+                    "Could not add blob for file download {}, error {}",
+                    filename,
+                    err
+                )
+            })?;
+        blob.as_name().to_string()
+    };
+    info!(context, "saved download to: {:?}", saved_path);
+
+    // TODO: Support getting the mime type.
+    let filemime = None;
+
+    message.set_file(saved_path, filemime);
+    message.save_param_to_disk(context).await;
+
+    Ok(())
+}
+
+/// Download and decrypt a file from a HTTP endpoint.
+pub async fn download_file(
+    context: &Context,
+    url: impl AsRef<str>,
+    passphrase: String,
+) -> Result<Vec<u8>> {
+    info!(context, "downloading file from {}", &url.as_ref());
+    let response = surf::get(url).recv_bytes().await;
+    if let Err(err) = response {
+        bail!("Download failed: {}", err);
+    }
+    let bytes = response.unwrap();
+    info!(context, "download complete, len: {}", bytes.len());
+    let reader = Cursor::new(bytes);
+    let decrypted = symm_decrypt_bytes(&passphrase, reader).await?;
+    Ok(decrypted)
+}
+
+/// Parse a URL from a string and take out the hash fragment.
+fn parse_upload_url(url: impl AsRef<str>) -> Result<(String, Url)> {
+    let mut url = url::Url::parse(url.as_ref())?;
+    let passphrase = url.fragment();
+    if passphrase.is_none() {
+        bail!("Missing passphrase for upload URL");
+    }
+    let passphrase = passphrase.unwrap().to_string();
+    url.set_fragment(None);
+    Ok((passphrase, url))
+}
+
+/// Generate a random URL based on the provided endpoint.
+pub fn generate_upload_url(_context: &Context, mut endpoint: String) -> String {
+    // equals at least 16 random bytes (base32 takes 160% of binary size).
+    const FILENAME_LEN: usize = 26;
+    // equals at least 32 random bytes.
+    const PASSPHRASE_LEN: usize = 52;
+
+    if endpoint.ends_with('/') {
+        endpoint.pop();
+    }
+    let passphrase = generate_token_string(PASSPHRASE_LEN);
+    let filename = generate_token_string(FILENAME_LEN);
+    format!("{}/{}#{}", endpoint, filename, passphrase)
+}
+
+/// Generate a random string encoded in base32.
+/// Len is the desired string length of the result.
+/// TODO: There's likely better methods to create random tokens.
+pub fn generate_token_string(len: usize) -> String {
+    const CROCKFORD_ALPHABET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz";
+    let mut rng = rand::thread_rng();
+    let token: String = (0..len)
+        .map(|_| {
+            let idx = rng.gen_range(0, CROCKFORD_ALPHABET.len());
+            CROCKFORD_ALPHABET[idx] as char
+        })
+        .collect();
+    token
+}

upload-server/.gitignore

@@ -0,0 +1,4 @@
+uploads
+node_modules
+yarn*
+.gitfoo

upload-server/README.md

@@ -0,0 +1,17 @@
+# deltachat-upload-server
+
+Demo server for the HTTP file upload feature.
+
+### Usage
+
+```
+npm install
+node server.js
+```
+
+Configure with environment variables:
+* `UPLOAD_PATH`: Path to upload files to (default: `./uploads`)
+* `PORT`: Port to listen on (default: `8080`)
+* `HOSTNAME`: Hostname to listen on (default: `0.0.0.0`)
+* `BASEURL`: Base URL for generated links (default: `http://[hostname]:[port]/`)
+

upload-server/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "deltachat-upload-server",
+  "version": "1.0.0",
+  "main": "server.js",
+  "license": "MIT",
+  "scripts": {
+    "start": "node server.js"
+  },
+  "dependencies": {
+    "base32": "^0.0.6",
+    "express": "^4.17.1"
+  }
+}

upload-server/server.js

@@ -0,0 +1,73 @@
+const p = require('path')
+const express = require('express')
+const fs = require('fs')
+const { pipeline } = require('stream')
+
+const app = express()
+
+const config = {
+  path: process.env.UPLOAD_PATH || p.resolve('./uploads'),
+  port: process.env.PORT || 8080,
+  hostname: process.env.HOSTNAME || '0.0.0.0',
+  baseurl: process.env.BASE_URL
+}
+
+if (!config.baseurl) config.baseurl = `http://${config.hostname}:${config.port}/`
+if (!config.baseurl.endsWith('/')) config.baseurl = config.baseurl + '/'
+
+if (!fs.existsSync(config.path)) {
+  fs.mkdirSync(config.path, { recursive: true })
+}
+
+app.use('/:filename', checkFilenameMiddleware)
+app.put('/:filename', (req, res) => {
+  const uploadpath = req.uploadpath
+  const filename = req.params.filename
+  fs.stat(uploadpath, (err, stat) => {
+    if (err && err.code !== 'ENOENT') {
+      console.error('error', err.message)
+      return res.code(500).send('internal server error')
+    }
+    if (stat) return res.status(500).send('filename in use')
+
+    const ws = fs.createWriteStream(uploadpath)
+    pipeline(req, ws, err => {
+      if (err) {
+        console.error('error', err.message)
+        return res.status(500).send('internal server error')
+      }
+      console.log('file uploaded: ' + uploadpath)
+      const url = config.baseurl + filename
+      res.end(url)
+    })
+  })
+})
+
+app.get('/:filename', (req, res) => {
+  const uploadpath = req.uploadpath
+  const rs = fs.createReadStream(uploadpath)
+  res.setHeader('content-type', 'application/octet-stream')
+  pipeline(rs, res, err => {
+    if (err) console.error('error', err.message)
+    if (err) return res.status(500).send(err.message)
+  })
+})
+
+function checkFilenameMiddleware (req, res, next) {
+  const filename = req.params.filename
+  if (!filename) return res.status(500).send('missing filename')
+  if (!filename.match(/^[a-zA-Z0-9]{26,32}$/)) {
+    return res.status(500).send('illegal filename')
+  }
+  const uploadpath = p.normalize(p.join(config.path, req.params.filename))
+  if (!uploadpath.startsWith(config.path)) {
+    return res.code(500).send('bad request')
+  }
+  req.uploadpath = uploadpath
+  next()
+}
+
+app.listen(config.port, err => {
+  if (err) console.error(err)
+  else console.log(`Listening on ${config.baseurl}`)
+})