bump version to 1.40

Parallel processing of messages results in bugs such as messages sent by
a new member being processed before the message that adds this member to
the chat, even when it has lower UID. In this case, messages are shown as
"[Unknown sender for this chat. See 'info' for more details.]", while
there is a message "Member ... added" right before, because writing the
information about new member to the database takes longer then reading
old index from the database.

.circleci/config.yml

@@ -138,12 +138,6 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests_rust:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_rust.sh
-    
   remote_tests_python:
     machine: true
     steps:
@@ -178,11 +172,6 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
-
       - remote_tests_python:
           filters:
             tags:
@@ -191,8 +180,9 @@ workflows:
       - remote_python_packaging:
           requires: 
              - remote_tests_python
-             - remote_tests_rust 
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 
@@ -201,6 +191,8 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -212,6 +204,8 @@ workflows:
 
       - build_doxygen:
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: Rust CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.43.1
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.43.1
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly, 1.43.1]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      with:
+        command:  check
+        args: --workspace --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --workspace
+          

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-19
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

CHANGELOG.md

@@ -1,5 +1,102 @@
 # Changelog 
 
+## 1.40.0
+
+- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
+
+- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
+
+- improve idle #1690 #1688
+
+- refactorings #1670 #1673
+
+
+## 1.39.0
+
+- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
+
+- fix potential panics, update dependencies #1650 #1655
+
+
+## 1.38.0
+
+- fix sorting, esp. for multi-device
+
+
+## 1.37.0
+
+- improve ndn heuristics #1630
+
+- get oauth2 authorizer from provider-db #1641
+
+- removed linebreaks and spaces from generated qr-code #1631
+
+- more fixes #1633 #1635 #1636 #1637
+
+
+## 1.36.0
+
+- parse ndn (network delivery notification) reports
+  and report failed messages as such #1552 #1622 #1630
+
+- add oauth2 support for gsuite domains #1626
+
+- read image orientation from exif before recoding #1619
+
+- improve logging #1593 #1598
+
+- improve python and bot bindings #1583 #1609
+
+- improve imap logout #1595
+
+- fix sorting #1600 #1604
+
+- fix qr code generation #1631
+
+- update rustcrypto releases #1603
+
+- refactorings #1617
+
+
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
+## 1.34.0
+
+- new api for io, thread and event handling #1356,
+  see the example atop of `deltachat.h` to get an overview
+
+- LOTS of speed improvements due to async processing #1356
+
+- enable WAL mode for sqlite #1492
+
+- process incoming messages in bulk #1527
+
+- improve finding out the sent-folder #1488
+
+- several bug fixes
+
+
 ## 1.33.0
 
 - let `dc_set_muted()` also mute one-to-one chats #1470

Cargo.toml

@@ -1,66 +1,67 @@
 [package]
 name = "deltachat"
-version = "1.33.0"
+version = "1.40.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
 [profile.release]
-# lto = true
+#lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
 async-smtp = { version = "0.3" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.3.0"
+async-imap = "0.3.1"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.0", features = ["unstable"] }
-base64 = "0.11"
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
+kamadak-exif = "0.5"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.22", features = ["bundled"] }
-r2d2_sqlite = "0.15.0"
+rusqlite = { version = "0.23", features = ["bundled"] }
+r2d2_sqlite = "0.16.0"
 r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
+strum = "0.18.0"
+strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.17.1"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.0"
+mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
+async-std-resolver = "0.19.5"
 
-pretty_env_logger = { version = "0.3.1", optional = true }
+pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
@@ -69,9 +70,10 @@ ansi_term = { version = "0.12.1", optional = true }
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
 async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+smol = "0.1.10"
 
 [workspace]
 members = [
@@ -82,6 +84,7 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
+required-features = ["repl"]
 
 [[example]]
 name = "repl"

README.md

@@ -123,6 +123,7 @@ Language bindings are available for:
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
 - [Go](https://github.com/hugot/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library
@@ -132,4 +133,5 @@ or its language bindings:
 - [iOS](https://github.com/deltachat/deltachat-ios)
 - [Desktop](https://github.com/deltachat/deltachat-desktop)
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - several **Bots**

appveyor.yml

@@ -1,19 +0,0 @@
-environment:
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-19
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - rustc -vV
-  - cargo -vV
-
-build: false
-
-test_script:
-  - cargo test --release --all
-
-cache:
-  - target
-  - C:\Users\appveyor\.cargo\registry

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.33.0"
+version = "1.40.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -32,63 +32,40 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  *
  * Let's start.
  *
- * First of all, you have to **define an event-handler-function**
- * that is called by the library on specific events
- * (eg. when the configuration is done or when fresh messages arrive).
- * With this function you can create a Delta Chat context then:
+ * First of all, you have to **create a context object**
+ * bound to a database.
+ * The database is a normal sqlite-file and is created as needed:
  *
  * ~~~
- * #include <deltachat.h>
+ * dc_context_t* context = dc_context_new(NULL, "example.db", NULL);
+ * ~~~
  *
- * uintptr_t event_handler_func(dc_context_t* context, int event,
- *                              uintptr_t data1, uintptr_t data2)
+ * After that, make sure, you can **receive events from the context**.
+ * For that purpose, create an event emitter you can ask for events.
+ * If there are no event, the emitter will wait until there is one,
+ * so, in many situations you will do this in a thread:
+ *
+ * ~~~
+ * void* event_handler(void* context)
  * {
- *     return 0;
- * }
- *
- * dc_context_t* context = dc_context_new(event_handler_func, NULL, NULL);
- * ~~~
- *
- * After that, you should make sure,
- * sending and receiving jobs are processed as needed.
- * For this purpose, you have to **create two threads:**
- *
- * ~~~
- * #include <pthread.h>
- *
- * void* imap_thread_func(void* context)
- * {
- *     while (true) {
- *         dc_perform_imap_jobs(context);
- *         dc_perform_imap_fetch(context);
- *         dc_perform_imap_idle(context);
+ *     dc_event_emitter_t* emitter = dc_get_event_emitter(context);
+ *     dc_event_t* event;
+ *     while ((event = dc_get_next_event(emitter)) != NULL) {
+ *         // use the event as needed, eg. dc_event_get_id() returns the type.
+ *         // once you're done, unref the event to avoid memory leakage:
+ *         dc_event_unref(event);
  *     }
+ *     dc_event_emitter_unref(emitter);
  * }
  *
- * void* smtp_thread_func(void* context)
- * {
- *     while (true) {
- *         dc_perform_smtp_jobs(context);
- *         dc_perform_smtp_idle(context);
- *     }
- * }
- *
- * static pthread_t imap_thread, smtp_thread;
- * pthread_create(&imap_thread, NULL, imap_thread_func, context);
- * pthread_create(&smtp_thread, NULL, smtp_thread_func, context);
+ * static pthread_t event_thread;
+ * pthread_create(&event_thread, NULL, event_handler, context);
  * ~~~
  *
  * The example above uses "pthreads",
  * however, you can also use anything else for thread handling.
  * All deltachat-core-functions, unless stated otherwise, are thread-safe.
  *
- * After that you can  **define and open a database.**
- * The database is a normal sqlite-file and is created as needed:
- *
- * ~~~
- * dc_open(context, "example.db", NULL);
- * ~~~
- *
  * Now you can **configure the context:**
  *
  * ~~~
@@ -98,15 +75,22 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  * dc_configure(context);
  * ~~~
  *
- * dc_configure() returns immediately, the configuration itself may take a while
- * and is done by a job in the imap-thread you've defined above.
+ * dc_configure() returns immediately,
+ * the configuration itself runs in background and may take a while.
  * Once done, the #DC_EVENT_CONFIGURE_PROGRESS reports success
- * to the event_handler_func() that is also defined above.
+ * to the event_handler() you've defined above.
  *
  * The configuration result is saved in the database,
  * on subsequent starts it is not needed to call dc_configure()
  * (you can check this using dc_is_configured()).
  *
+ * On a successfully configured context,
+ * you can finally **connect to the servers:**
+ *
+ * ~~~
+ * dc_start_io(context);
+ * ~~~
+ *
  * Now you can **send the first message:**
  *
  * ~~~
@@ -118,11 +102,11 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  * ~~~
  *
  * dc_send_text_msg() returns immediately;
- * the sending itself is done by a job in the smtp-thread you've defined above.
+ * the sending itself is done in the background.
  * If you check the testing address (bob)
  * and you should have received a normal email.
  * Answer this email in any email program with "Got it!"
- * and the imap-thread you've create above will **receive the message**.
+ * and the IO you started above will **receive the message**.
  *
  * You can then **list all messages** of a chat as follow:
  *
@@ -166,19 +150,6 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  * - The issue-tracker for the core library is here:
  *   <https://github.com/deltachat/deltachat-core-rust/issues>
  *
- * The following points are important mainly
- * for the authors of the library itself:
- *
- * - For indentation, use tabs.
- *   Alignments that are not placed at the beginning of a line
- *   should be done with spaces.
- *
- * - For padding between functions,
- *   classes etc. use 2 empty lines
- *
- * - Source files are encoded as UTF-8 with Unix line endings
- *   (a simple `LF`, `0x0A` or `\n`)
- *
  * If you need further assistance,
  * please do not hesitate to contact us
  * through the channels shown at https://delta.chat/en/contribute
@@ -191,24 +162,6 @@ typedef struct _dc_event_emitter dc_event_emitter_t;
  */
 
 
-/**
- * TODO: document
- */  
-dc_event_t*   dc_get_next_event(dc_event_emitter_t* emitter);
-
-dc_event_emitter_t* dc_get_event_emitter(dc_context_t* context);
-void  dc_event_emitter_unref(dc_event_emitter_t* emitter);
-
-int           dc_event_get_id   (dc_event_t* event);
-int           dc_event_get_data1_int(dc_event_t* event);
-int           dc_event_get_data2_int(dc_event_t* event);
-char*         dc_event_get_data3_str(dc_event_t* event);
-
-/**
- * TODO: document
- */
-void          dc_event_unref   (dc_event_t* event);
-
 /**
  * @class dc_context_t
  *
@@ -226,21 +179,6 @@ void          dc_event_unref   (dc_event_t* event);
  * opened, connected and mails are fetched.
  *
  * @memberof dc_context_t
- * @param cb a callback function that is called for events (update,
- *     state changes etc.) and to get some information from the client (eg. translation
- *     for a given string).
- *     See @ref DC_EVENT for a list of possible events that may be passed to the callback.
- *     - The callback MAY be called from _any_ thread, not only the main/GUI thread!
- *     - The callback MUST NOT call any dc_* and related functions unless stated
- *       otherwise!
- *     - The callback SHOULD return _fast_, for GUI updates etc. you should
- *       post yourself an asynchronous message to your GUI thread, if needed.
- *     - events do not expect a return value, just always return 0.
- * @param dbfile The file to use to store the database, something like `~/file` won't
- *     work on all systems, if in doubt, use absolute paths.
- * @param blobdir A directory to store the blobs in; a trailing slash is not needed.
- *     If you pass NULL or the empty string, deltachat-core creates a directory
- *     beside _dbfile_ with the same name and the suffix `-blobs`.
  * @param os_name is only for decorative use
  *     and is shown eg. in the `X-Mailer:` header
  *     in the form "Delta Chat Core <version>/<os_name>".
@@ -248,6 +186,11 @@ void          dc_event_unref   (dc_event_t* event);
  *     the used environment and/or the version here.
  *     It is okay to give NULL, in this case `X-Mailer:` header
  *     is set to "Delta Chat Core <version>".
+ * @param dbfile The file to use to store the database,
+ *     something like `~/file` won't work, use absolute paths.
+ * @param blobdir A directory to store the blobs in; a trailing slash is not needed.
+ *     If you pass NULL or the empty string, deltachat-core creates a directory
+ *     beside _dbfile_ with the same name and the suffix `-blobs`.
  * @return A context object with some public members.
  *     The object must be passed to the other context functions
  *     and must be freed using dc_context_unref() after usage.
@@ -269,6 +212,25 @@ dc_context_t*   dc_context_new               (const char* os_name, const char* d
  */
 void            dc_context_unref             (dc_context_t* context);
 
+
+/**
+ * Create the event emitter that is used to receive events.
+ * The library will emit various @ref DC_EVENT events as "new message", "message read" etc.
+ * To get these events, you have to create an event emitter using this function
+ * and call dc_get_next_event() on the emitter.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as created by dc_context_new().
+ * @return Returns the event emitter, NULL on errors.
+ *     Must be freed using dc_event_emitter_unref() after usage.
+ *
+ * Note: Use only one event emitter per context.
+ * Having more than one event emitter running at the same time on the same context
+ * will result in events randomly delivered to the one or to the other.
+ */
+dc_event_emitter_t* dc_get_event_emitter(dc_context_t* context);
+
+
 /**
  * Get the blob directory.
  *
@@ -300,17 +262,25 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
  *                    `blobdir`; the original image will not be needed anymore.
  *                    NULL to remove the avatar.
- *                    It is planned for future versions
- *                    to send this image together with the next messages.
+ *                    As for `displayname` and `selfstatus`, also the avatar is sent to the recipients.
+ *                    To save traffic, however, the avatar is attached only as needed
+ *                    and also recoded to a reasonable size.
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
  *                    1=send and request read receipts (default)
+ * - `bcc_self`     = 0=do not send a copy of outgoing messages to self (default),
+ *                    1=send a copy of outgoing messages to self.
+ *                    Sending messages to self is needed for a proper multi-account setup,
+ *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
  * - `inbox_watch`  = 1=watch `INBOX`-folder for changes (default),
- *                    0=do not watch the `INBOX`-folder
+ *                    0=do not watch the `INBOX`-folder,
+ *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes (default),
- *                    0=do not watch the `Sent`-folder
+ *                    0=do not watch the `Sent`-folder,
+ *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `mvbox_watch`  = 1=watch `DeltaChat`-folder for changes (default),
- *                    0=do not watch the `DeltaChat`-folder
+ *                    0=do not watch the `DeltaChat`-folder,
+ *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `mvbox_move`   = 1=heuristically detect chat-messages
  *                    and move them to the `DeltaChat`-folder,
  *                    0=do not move chat-messages
@@ -345,9 +315,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    DC_MEDIA_QUALITY_WORSE (1)
  *                    allow worse images/videos/voice quality to gain smaller sizes,
  *                    suitable for providers or areas known to have a bad connection.
- *                    In contrast to other options, the implementation of this option is currently up to the UIs;
- *                    this may change in future, however,
- *                    having the option in the core allows provider-specific-defaults already today.
+ *                    The library uses the `media_quality` setting to use different defaults
+ *                    for recoding images sent with type DC_MSG_IMAGE.
+ *                    If needed, recoding other file types is up to the UI.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -468,9 +438,7 @@ char*           dc_get_oauth2_url            (dc_context_t* context, const char*
 
 /**
  * Configure a context.
- * For this purpose, the function creates a job
- * that is executed in the IMAP-thread then;
- * this requires to call dc_perform_imap_jobs() regularly.
+ * While configuration IO must not be started, if needed stop IO using dc_stop_io() first.
  * If the context is already configured,
  * this function will try to change the configuration.
  *
@@ -537,6 +505,8 @@ int             dc_is_configured   (const dc_context_t* context);
 
 /**
  * Start job and IMAP/SMTP tasks.
+ * If IO is already running, nothing happens.
+ * To check the current IO state, use dc_is_io_running().
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
@@ -556,6 +526,8 @@ int             dc_is_io_running(const dc_context_t* context);
 
 /**
  * Stop job and IMAP/SMTP tasks and return when they are finished. 
+ * If IO is not running, nothing happens.
+ * To check the current IO state, use dc_is_io_running().
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
@@ -564,9 +536,20 @@ int             dc_is_io_running(const dc_context_t* context);
 void            dc_stop_io(dc_context_t* context);
 
 /**
- * This function can be called whenever there is a hint
- * that the network is available again.
- * The library will try to send pending messages out.
+ * This function should be called when there is a hint
+ * that the network is available again,
+ * eg. as a response to system event reporting network availability.
+ * The library will try to send pending messages out immediately.
+ *
+ * Moreover, to have a reliable state
+ * when the app comes to foreground with network available,
+ * it may be reasonable to call the function also at that moment.
+ *
+ * It is okay to call the function unconditionally when there is
+ * network available, however, calling the function
+ * _without_ having network may interfere with the backoff algorithm
+ * and will led to let the jobs fail faster, with fewer retries
+ * and may avoid messages being sent out.
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
@@ -790,6 +773,15 @@ uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t ch
  * dc_msg_unref(msg);
  * ~~~
  *
+ * If you send images with the DC_MSG_IMAGE type,
+ * they will be recoded to a reasonable size before sending, if possible
+ * (cmp the dc_set_config()-option `media_quality`).
+ * If that fails, is not possible, or the image is already small enough, the image is sent as original.
+ * If you want images to be always sent as the original file, use the DC_MSG_FILE type.
+ *
+ * Videos and other file types are currently not recoded by the library,
+ * with dc_prepare_msg(), however, you can do that from the UI.
+ *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id Chat ID to send the message to.
@@ -985,6 +977,7 @@ dc_msg_t*       dc_get_draft                 (dc_context_t* context, uint32_t ch
  * @param chat_id The chat ID of which the messages IDs should be queried.
  * @param flags If set to DC_GCM_ADDDAYMARKER, the marker DC_MSG_ID_DAYMARKER will
  *     be added before each day (regarding the local timezone).  Set this to 0 if you do not want this behaviour.
+ *     To get the concrete time of the marker, use dc_array_get_timestamp().
  * @param marker1before An optional message ID.  If set, the id DC_MSG_ID_MARKER1 will be added just
  *   before the given ID in the returned array.  Set this to 0 if you do not want this behaviour.
  * @return Array of message IDs, must be dc_array_unref()'d when no longer used.
@@ -1179,6 +1172,16 @@ void            dc_delete_chat               (dc_context_t* context, uint32_t ch
  */
 dc_array_t*     dc_get_chat_contacts         (dc_context_t* context, uint32_t chat_id);
 
+/**
+ * Get the chat's ephemeral message timer.
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param chat_id The chat ID.
+ *
+ * @return ephemeral timer value in seconds, 0 if the timer is disabled or if there is an error
+ */
+uint32_t dc_get_chat_ephemeral_timer (dc_context_t* context, uint32_t chat_id);
 
 /**
  * Search messages containing the given query string.
@@ -1311,6 +1314,21 @@ int             dc_remove_contact_from_chat  (dc_context_t* context, uint32_t ch
  */
 int             dc_set_chat_name             (dc_context_t* context, uint32_t chat_id, const char* name);
 
+/**
+ * Set the chat's ephemeral message timer.
+ *
+ * This timer is applied to all messages in a chat and starts when the
+ * message is read. The setting is synchronized to all clients
+ * participating in a chat.
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param chat_id The chat ID to set the ephemeral message timer for.
+ * @param timer The timer value in seconds or 0 to disable the timer.
+ *
+ * @return 1=success, 0=error
+ */
+int dc_set_chat_ephemeral_timer (dc_context_t* context, uint32_t chat_id, uint32_t timer);
 
 /**
  * Set group profile image.
@@ -1674,9 +1692,6 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
 
 /**
  * Import/export things.
- * For this purpose, the function creates a job that is executed in the IMAP-thread then;
- * this requires to call dc_perform_imap_jobs() regularly.
- *
  * What to do is defined by the _what_ parameter which may be one of the following:
  *
  * - **DC_IMEX_EXPORT_BACKUP** (11) - Export a backup to the directory given as `param1`.
@@ -2296,17 +2311,6 @@ int              dc_array_is_independent     (const dc_array_t* array, size_t in
 int              dc_array_search_id          (const dc_array_t* array, uint32_t needle, size_t* ret_index);
 
 
-/**
- * Get raw pointer to the data.
- *
- * @memberof dc_array_t
- * @param array The array object.
- * @return Raw pointer to the array. You MUST NOT free the data. You MUST NOT access the data beyond the current item count.
- *     It is not possible to enlarge the array this way.  Calling any other dc_array*()-function may discard the returned pointer.
- */
-const uint32_t*  dc_array_get_raw            (const dc_array_t* array);
-
-
 /**
  * @class dc_chatlist_t
  *
@@ -2821,6 +2825,9 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_DELIVERED.
  * - DC_STATE_OUT_MDN_RCVD (28) - Outgoing message read by the recipient (two checkmarks; this requires goodwill on the receiver's side)
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_READ.
+ *   Also messages already read by some recipients
+ *   may get into the state DC_STATE_OUT_FAILED at a later point,
+ *   eg. when in a group, delivery fails for some recipients.
  *
  * If you just want to check if a message is sent or not, please use dc_msg_is_sent() which regards all states accordingly.
  *
@@ -3857,11 +3864,122 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EMPTY_INBOX 0x02 // Deprecated, flag for dc_empty_server(): Clear all INBOX messages
 
 
+/**
+ * @class dc_event_emitter_t
+ *
+ * Opaque object that is used to get events.
+ * You can get an event emitter from a context using dc_get_event_emitter().
+ */
+
+/**
+ * Get the next event from an event emitter object.
+ *
+ * @memberof dc_event_emitter_t
+ * @param emitter Event emitter object as returned from dc_get_event_emitter().
+ * @return An event as an dc_event_t object.
+ *     You can query the event for information using dc_event_get_id(), dc_event_get_data1_int() and so on;
+ *     if you are done with the event, you have to free the event using dc_event_unref().
+ *     If NULL is returned, the context belonging to the event emitter is unref'd and the no more events will come;
+ *     in this case, free the event emitter using dc_event_emitter_unref().
+ */
+dc_event_t* dc_get_next_event(dc_event_emitter_t* emitter);
+
+
+/**
+ * Free an event emitter object.
+ *
+ * @memberof dc_event_emitter_t
+ * @param emitter Event emitter object as returned from dc_get_event_emitter().
+ *     If NULL is given, nothing is done and an error is logged.
+ * @return None.
+ */
+void  dc_event_emitter_unref(dc_event_emitter_t* emitter);
+
+
+/**
+ * @class dc_event_t
+ *
+ * Opaque object describing a single event.
+ * To get events, call dc_get_next_event() on an event emitter created by dc_get_event_emitter().
+ */
+
+/**
+ * Get the event-id from an event object.
+ * The event-id is one of the @ref DC_EVENT constants.
+ * There may be additional data belonging to an event,
+ * to get them, use dc_event_get_data1_int(), dc_event_get_data2_int() and dc_event_get_data2_str().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return once of the @ref DC_EVENT constants.
+ *     0 on errors.
+ */
+int dc_event_get_id(dc_event_t* event);
+
+
+/**
+ * Get a data associated with an event object.
+ * The meaning of the data depends on the event-id
+ * returned as @ref DC_EVENT constants by dc_event_get_id().
+ * See also dc_event_get_data2_int() and dc_event_get_data2_str().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data1" as a signed integer, at least 32bit,
+ *     the meaning depends on the event type associated with this event.
+ */
+int dc_event_get_data1_int(dc_event_t* event);
+
+
+/**
+ * Get a data associated with an event object.
+ * The meaning of the data depends on the event-id
+ * returned as @ref DC_EVENT constants by dc_event_get_id().
+ * See also dc_event_get_data2_int() and dc_event_get_data2_str().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data2" as a signed integer, at least 32bit,
+ *     the meaning depends on the event type associated with this event.
+ */
+int dc_event_get_data2_int(dc_event_t* event);
+
+
+/**
+ * Get a data associated with an event object.
+ * The meaning of the data depends on the event-id
+ * returned as @ref DC_EVENT constants by dc_event_get_id().
+ * See also dc_event_get_data1_int() and dc_event_get_data2_int().
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data2" as a string,
+ *     the meaning depends on the event type associated with this event.
+ *     Once you're done with the string, you have to unref it using dc_unref_str().
+ */
+char* dc_event_get_data2_str(dc_event_t* event);
+
+
+/**
+ * Free memory used by an event object.
+ * If you forget to do this for an event, this will result in memory leakage.
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return None.
+ */
+void dc_event_unref(dc_event_t* event);
+
+
 /**
  * @defgroup DC_EVENT DC_EVENT
  *
- * These constants are used as events
- * reported to the callback given to dc_context_new().
+ * These constants are used as event-id
+ * in events returned by dc_get_next_event().
+ *
+ * Events typically come with some additional data,
+ * use dc_event_get_data1_int(), dc_event_get_data2_int() and dc_event_get_data2_str() to read this data.
+ * The meaning of the data depends on the event.
  *
  * @addtogroup DC_EVENT
  * @{
@@ -3869,13 +3987,11 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * The library-user may write an informational string to the log.
- * Passed to the callback given to dc_context_new().
  *
  * This event should not be reported to the end-user using a popup or something like that.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_INFO                     100
 
@@ -3884,8 +4000,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when SMTP connection is established and login was successful.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_SMTP_CONNECTED           101
 
@@ -3894,8 +4009,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when IMAP connection is established and login was successful.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_IMAP_CONNECTED           102
 
@@ -3903,8 +4017,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a message was successfully sent to the SMTP server.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_SMTP_MESSAGE_SENT        103
 
@@ -3912,8 +4025,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a message was successfully marked as deleted on the IMAP server.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_IMAP_MESSAGE_DELETED   104
 
@@ -3921,8 +4033,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a message was successfully moved on IMAP.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
@@ -3930,8 +4041,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when an IMAP folder was emptied.
  *
  * @param data1 0
- * @param data2 (const char*) folder name.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Folder name.
  */
 #define DC_EVENT_IMAP_FOLDER_EMPTIED  106
 
@@ -3939,8 +4049,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a new blob file was successfully written
  *
  * @param data1 0
- * @param data2 (const char*) path name
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Path name
  */
 #define DC_EVENT_NEW_BLOB_FILE 150
 
@@ -3948,27 +4057,23 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * Emitted when a blob file was successfully deleted
  *
  * @param data1 0
- * @param data2 (const char*) path name
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Path name
  */
 #define DC_EVENT_DELETED_BLOB_FILE 151
 
 /**
  * The library-user should write a warning string to the log.
- * Passed to the callback given to dc_context_new().
  *
  * This event should not be reported to the end-user using a popup or something like that.
  *
  * @param data1 0
- * @param data2 (const char*) Warning string in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Warning string in english language.
  */
 #define DC_EVENT_WARNING                  300
 
 
 /**
  * The library-user should report an error to the end-user.
- * Passed to the callback given to dc_context_new().
  *
  * As most things are asynchronous, things may go wrong at any time and the user
  * should not be disturbed by a dialog or so.  Instead, use a bubble or so.
@@ -3980,10 +4085,9 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * in a messasge box then.
  *
  * @param data1 0
- * @param data2 (const char*) Error string, always set, never NULL.
+ * @param data2 (char*) Error string, always set, never NULL.
  *     Some error strings are taken from dc_set_stock_translation(),
  *     however, most error strings will be in english language.
- *     Must not be unref'd or modified and is valid only until the callback returns.
  */
 #define DC_EVENT_ERROR                    400
 
@@ -4005,8 +4109,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 1=first/new network error, should be reported the user;
  *     0=subsequent network error, should be logged only
- * @param data2 (const char*) Error string, always set, never NULL.
- *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @param data2 (char*) Error string, always set, never NULL.
  */
 #define DC_EVENT_ERROR_NETWORK            401
 
@@ -4019,9 +4122,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * dc_send_text_msg() or another sending function.
  *
  * @param data1 0
- * @param data2 (const char*) Info string in english language.
- *     Must not be unref'd or modified
- *     and is valid only until the callback returns.
+ * @param data2 (char*) Info string in english language.
  */
 #define DC_EVENT_ERROR_SELF_NOT_IN_GROUP  410
 
@@ -4062,8 +4163,9 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 
 /**
- * A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
- * DC_STATE_OUT_FAILED, see dc_msg_get_state().
+ * A single message could not be sent.
+ * State changed from DC_STATE_OUT_PENDING, DC_STATE_OUT_DELIVERED or DC_STATE_OUT_MDN_RCVD
+ * to DC_STATE_OUT_FAILED, see dc_msg_get_state().
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -4092,6 +4194,11 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define DC_EVENT_CHAT_MODIFIED            2020
 
+/**
+ * Chat ephemeral timer changed.
+ */
+#define DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED 2021
+
 
 /**
  * Contact(s) created, renamed, verified, blocked or deleted.
@@ -4139,9 +4246,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * A typical purpose for a handler of this event may be to make the file public to some system
  * services.
  *
- * @param data1 (const char*) Path and file name.
- *     Must not be unref'd or modified and is valid only until the callback returns.
- * @param data2 0
+ * @param data1 0
+ * @param data2 (char*) Path and file name.
  */
 #define DC_EVENT_IMEX_FILE_WRITTEN        2052
 
@@ -4229,7 +4335,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 /**
- * Prover works out-of-the-box.
+ * Provider works out-of-the-box.
  * This provider status is returned for provider where the login
  * works by just entering the name or the email-address.
  *
@@ -4369,12 +4475,33 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_STR_LOCATION                   66
 #define DC_STR_STICKER                    67
 #define DC_STR_DEVICE_MESSAGES            68
-#define DC_STR_COUNT                      68
+#define DC_STR_SAVED_MESSAGES             69
+#define DC_STR_DEVICE_MESSAGES_HINT       70
+#define DC_STR_WELCOME_MESSAGE            71
+#define DC_STR_UNKNOWN_SENDER_FOR_CHAT    72
+#define DC_STR_SUBJECT_FOR_NEW_CONTACT    73
+#define DC_STR_FAILED_SENDING_TO          74
+#define DC_STR_EPHEMERAL_DISABLED         75
+#define DC_STR_EPHEMERAL_SECONDS          76
+#define DC_STR_EPHEMERAL_MINUTE           77
+#define DC_STR_EPHEMERAL_HOUR             78
+#define DC_STR_EPHEMERAL_DAY              79
+#define DC_STR_EPHEMERAL_WEEK             80
+#define DC_STR_EPHEMERAL_FOUR_WEEKS       81
+
+#define DC_STR_COUNT                      81
 
 /*
  * @}
  */
 
+#ifdef PY_CFFI_INC
+/* Helper utility to locate the header file when building python bindings. */
+char* _dc_header_file_location(void) {
+    return __FILE__;
+}
+#endif
+
 
 #ifdef __cplusplus
 }

deltachat-ffi/src/dc_array.rs

@@ -1,46 +1,56 @@
+use crate::chat::ChatItem;
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
+use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
+    MsgIds(Vec<MsgId>),
+    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub fn new(capacity: usize) -> Self {
-        dc_array_t::Uint(Vec::with_capacity(capacity))
-    }
-
-    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
-    pub fn new_locations(capacity: usize) -> Self {
-        dc_array_t::Locations(Vec::with_capacity(capacity))
-    }
-
-    pub fn add_id(&mut self, item: u32) {
-        if let Self::Uint(array) = self {
-            array.push(item);
-        } else {
-            panic!("Attempt to add id to array of other type");
-        }
-    }
-
-    pub fn add_location(&mut self, location: Location) {
-        if let Self::Locations(array) = self {
-            array.push(location)
-        } else {
-            panic!("Attempt to add a location to array of other type");
-        }
-    }
-
-    pub fn get_id(&self, index: usize) -> u32 {
+    pub(crate) fn get_id(&self, index: usize) -> u32 {
         match self {
+            Self::MsgIds(array) => array[index].to_u32(),
+            Self::Chat(array) => match array[index] {
+                ChatItem::Message { msg_id } => msg_id.to_u32(),
+                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
+                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
+            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index] as u32,
+            Self::Uint(array) => array[index],
         }
     }
 
-    pub fn get_location(&self, index: usize) -> &Location {
+    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(array) => array.get(index).and_then(|item| match item {
+                ChatItem::Message { .. } => None,
+                ChatItem::Marker1 { .. } => None,
+                ChatItem::DayMarker { timestamp } => Some(*timestamp),
+            }),
+            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(_) => None,
+            Self::Locations(array) => array
+                .get(index)
+                .and_then(|location| location.marker.as_deref()),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -48,55 +58,18 @@ impl dc_array_t {
         }
     }
 
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Locations(array) => array.is_empty(),
-            Self::Uint(array) => array.is_empty(),
-        }
-    }
-
     /// Returns the number of elements in the array.
-    pub fn len(&self) -> usize {
+    pub(crate) fn len(&self) -> usize {
         match self {
+            Self::MsgIds(array) => array.len(),
+            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub fn clear(&mut self) {
-        match self {
-            Self::Locations(array) => array.clear(),
-            Self::Uint(array) => array.clear(),
-        }
-    }
-
-    pub fn search_id(&self, needle: u32) -> Option<usize> {
-        if let Self::Uint(array) = self {
-            for (i, &u) in array.iter().enumerate() {
-                if u == needle {
-                    return Some(i);
-                }
-            }
-            None
-        } else {
-            panic!("Attempt to search for id in array of other type");
-        }
-    }
-
-    pub fn sort_ids(&mut self) {
-        if let dc_array_t::Uint(v) = self {
-            v.sort();
-        } else {
-            panic!("Attempt to sort array of something other than uints");
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const u32 {
-        if let dc_array_t::Uint(v) = self {
-            v.as_ptr()
-        } else {
-            panic!("Attempt to convert array of something other than uints to raw");
-        }
+    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
+        (0..self.len()).find(|i| self.get_id(*i) == needle)
     }
 }
 
@@ -106,6 +79,18 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
+impl From<Vec<MsgId>> for dc_array_t {
+    fn from(array: Vec<MsgId>) -> Self {
+        dc_array_t::MsgIds(array)
+    }
+}
+
+impl From<Vec<ChatItem>> for dc_array_t {
+    fn from(array: Vec<ChatItem>) -> Self {
+        dc_array_t::Chat(array)
+    }
+}
+
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -118,12 +103,11 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let mut arr = dc_array_t::new(7);
-        assert!(arr.is_empty());
+        let arr: dc_array_t = Vec::<u32>::new().into();
+        assert!(arr.len() == 0);
 
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
 
         assert_eq!(arr.len(), 1000);
 
@@ -131,31 +115,15 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        arr.clear();
-
-        assert!(arr.is_empty());
-
-        arr.add_id(13);
-        arr.add_id(7);
-        arr.add_id(666);
-        arr.add_id(0);
-        arr.add_id(5000);
-
-        arr.sort_ids();
-
-        assert_eq!(arr.get_id(0), 0);
-        assert_eq!(arr.get_id(1), 7);
-        assert_eq!(arr.get_id(2), 13);
-        assert_eq!(arr.get_id(3), 666);
+        assert_eq!(arr.search_id(10), Some(8));
+        assert_eq!(arr.search_id(1), None);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let mut arr = dc_array_t::new(7);
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/lib.rs

@@ -27,6 +27,7 @@ use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
+use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock::StockMessage;
@@ -126,7 +127,7 @@ pub unsafe extern "C" fn dc_set_config(
         // When ctx.set_config() fails it already logged the error.
         // TODO: Context::set_config() should not log this
         Ok(key) => block_on(async move {
-            ctx.set_config(key, to_opt_string_lossy(value).as_ref().map(|x| x.as_str()))
+            ctx.set_config(key, to_opt_string_lossy(value).as_deref())
                 .await
                 .is_ok() as libc::c_int
         }),
@@ -285,7 +286,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     }
     let ctx = &*context;
 
-    block_on({ ctx.start_io() })
+    block_on(ctx.start_io())
 }
 
 #[no_mangle]
@@ -295,7 +296,7 @@ pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c
     }
     let ctx = &*context;
 
-    block_on({ ctx.is_io_running() }) as libc::c_int
+    block_on(ctx.is_io_running()) as libc::c_int
 }
 
 #[no_mangle]
@@ -349,7 +350,8 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | Event::MsgDelivered { chat_id, .. }
         | Event::MsgFailed { chat_id, .. }
         | Event::MsgRead { chat_id, .. }
-        | Event::ChatModified(chat_id) => chat_id.to_u32() as libc::c_int,
+        | Event::ChatModified(chat_id)
+        | Event::ChatEphemeralTimerModified { chat_id, .. } => chat_id.to_u32() as libc::c_int,
         Event::ContactsChanged(id) | Event::LocationChanged(id) => {
             let id = id.unwrap_or_default();
             id as libc::c_int
@@ -399,13 +401,14 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | Event::MsgRead { msg_id, .. } => msg_id.to_u32() as libc::c_int,
         Event::SecurejoinInviterProgress { progress, .. }
         | Event::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
+        Event::ChatEphemeralTimerModified { timer, .. } => *timer as libc::c_int,
     }
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_event_get_data3_str(event: *mut dc_event_t) -> *mut libc::c_char {
+pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut libc::c_char {
     if event.is_null() {
-        eprintln!("ignoring careless call to dc_event_get_data3_str()");
+        eprintln!("ignoring careless call to dc_event_get_data2_str()");
         return ptr::null_mut();
     }
 
@@ -439,7 +442,8 @@ pub unsafe extern "C" fn dc_event_get_data3_str(event: *mut dc_event_t) -> *mut
         | Event::ConfigureProgress(_)
         | Event::ImexProgress(_)
         | Event::SecurejoinInviterProgress { .. }
-        | Event::SecurejoinJoinerProgress { .. } => ptr::null_mut(),
+        | Event::SecurejoinJoinerProgress { .. }
+        | Event::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         Event::ImexFileWritten(file) => {
             let data2 = file.to_c_string().unwrap_or_default();
             data2.into_raw()
@@ -554,14 +558,7 @@ pub unsafe extern "C" fn dc_get_chatlist(
     let qi = if query_id == 0 { None } else { Some(query_id) };
 
     block_on(async move {
-        match chatlist::Chatlist::try_load(
-            &ctx,
-            flags as usize,
-            qs.as_ref().map(|x| x.as_str()),
-            qi,
-        )
-        .await
-        {
+        match chatlist::Chatlist::try_load(&ctx, flags as usize, qs.as_deref(), qi).await {
             Ok(list) => {
                 let ffi_list = ChatlistWrapper { context, list };
                 Box::into_raw(Box::new(ffi_list))
@@ -752,13 +749,9 @@ pub unsafe extern "C" fn dc_add_device_msg(
     };
 
     block_on(async move {
-        chat::add_device_msg(
-            &ctx,
-            to_opt_string_lossy(label).as_ref().map(|x| x.as_str()),
-            msg,
-        )
-        .await
-        .unwrap_or_log_default(&ctx, "Failed to add device message")
+        chat::add_device_msg(&ctx, to_opt_string_lossy(label).as_deref(), msg)
+            .await
+            .unwrap_or_log_default(&ctx, "Failed to add device message")
     })
     .to_u32()
 }
@@ -841,14 +834,11 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
     };
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_msgs(&ctx, ChatId::new(chat_id), flags, marker_flag)
                 .await
-                .iter()
-                .map(|msg_id| msg_id.to_u32())
-                .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+                .into(),
+        ))
     })
 }
 
@@ -977,7 +967,7 @@ pub unsafe extern "C" fn dc_get_chat_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_media(
                 &ctx,
                 ChatId::new(chat_id),
@@ -986,11 +976,8 @@ pub unsafe extern "C" fn dc_get_chat_media(
                 or_msg_type3,
             )
             .await
-            .iter()
-            .map(|msg_id| msg_id.to_u32())
-            .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+            .into(),
+        ))
     })
 }
 
@@ -1296,6 +1283,49 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    // Timer value 0 is returned in the rare case of a database error,
+    // but it is not dangerous since it is only meant to be used as a
+    // default when changing the value. Such errors should not be
+    // ignored when ephemeral timer value is used to construct
+    // message headers.
+    block_on(async move { ChatId::new(chat_id).get_ephemeral_timer(ctx).await })
+        .log_err(ctx, "Failed to get ephemeral timer")
+        .unwrap_or_default()
+        .to_u32()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    timer: u32,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_set_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .set_ephemeral_timer(ctx, EphemeralTimer::from_u32(timer))
+            .await
+            .log_err(ctx, "Failed to set ephemeral timer")
+            .is_ok() as libc::c_int
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_msg_info(
     context: *mut dc_context_t,
@@ -1829,7 +1859,11 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
     }
     let ctx = &*context;
 
-    block_on({ location::send_locations_to_chat(&ctx, ChatId::new(chat_id), seconds as i64) });
+    block_on(location::send_locations_to_chat(
+        &ctx,
+        ChatId::new(chat_id),
+        seconds as i64,
+    ));
 }
 
 #[no_mangle]
@@ -1987,7 +2021,7 @@ pub unsafe extern "C" fn dc_array_get_timestamp(
         return 0;
     }
 
-    (*array).get_location(index).timestamp
+    (*array).get_timestamp(index).unwrap_or_default()
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_chat_id(
@@ -2034,7 +2068,7 @@ pub unsafe extern "C" fn dc_array_get_marker(
         return std::ptr::null_mut(); // NULL explicitly defined as "no markers"
     }
 
-    if let Some(s) = &(*array).get_location(index).marker {
+    if let Some(s) = (*array).get_marker(index) {
         s.strdup()
     } else {
         std::ptr::null_mut()
@@ -2062,16 +2096,6 @@ pub unsafe extern "C" fn dc_array_search_id(
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_array_get_raw(array: *const dc_array_t) -> *const u32 {
-    if array.is_null() {
-        eprintln!("ignoring careless call to dc_array_get_raw()");
-        return ptr::null_mut();
-    }
-
-    (*array).as_ptr()
-}
-
 // Return the independent-state of the location at the given index.
 // Independent locations do not belong to the track of the user.
 // Returns 1 if location belongs to the track of the user,
@@ -2812,7 +2836,7 @@ pub unsafe extern "C" fn dc_msg_set_file(
     let ffi_msg = &mut *msg;
     ffi_msg.message.set_file(
         to_string_lossy(file),
-        to_opt_string_lossy(filemime).as_ref().map(|x| x.as_str()),
+        to_opt_string_lossy(filemime).as_deref(),
     )
 }
 

examples/repl/cmdline.rs

@@ -488,6 +488,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
+            let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
                 &context,
                 listflags,
@@ -495,6 +496,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
+            let time_needed = std::time::SystemTime::now()
+                .duration_since(time_start)
+                .unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -553,6 +557,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
+            println!("{:?} to create this list", time_needed);
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {

examples/repl/main.rs

@@ -146,10 +146,8 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 11] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
-    "open",
-    "close",
     "set",
     "get",
     "oauth2",
@@ -290,48 +288,59 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let h = DcHelper {
-        completer: FilenameCompleter::new(),
-        highlighter: MatchingBracketHighlighter::new(),
-        hinter: HistoryHinter {},
-    };
-    let mut rl = Editor::with_config(config);
-    rl.set_helper(Some(h));
-    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
-    if rl.load_history(".dc-history.txt").is_err() {
-        println!("No previous history.");
-    }
-
     let mut selected_chat = ChatId::default();
+    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let input_loop = async_std::task::spawn_blocking(move || {
+        let h = DcHelper {
+            completer: FilenameCompleter::new(),
+            highlighter: MatchingBracketHighlighter::new(),
+            hinter: HistoryHinter {},
+        };
+        let mut rl = Editor::with_config(config);
+        rl.set_helper(Some(h));
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        if rl.load_history(".dc-history.txt").is_err() {
+            println!("No previous history.");
+        }
 
-    loop {
-        let p = "> ";
-        let readline = rl.readline(&p);
-        match readline {
-            Ok(line) => {
-                // TODO: ignore "set mail_pw"
-                rl.add_history_entry(line.as_str());
-                match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-                    Ok(ExitResult::Continue) => {}
-                    Ok(ExitResult::Exit) => break,
-                    Err(err) => println!("Error: {}", err),
+        loop {
+            let p = "> ";
+            let readline = rl.readline(&p);
+
+            match readline {
+                Ok(line) => {
+                    // TODO: ignore "set mail_pw"
+                    rl.add_history_entry(line.as_str());
+                    async_std::task::block_on(reader_s.send(line));
+                }
+                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                    println!("Exiting...");
+                    drop(reader_s);
+                    break;
+                }
+                Err(err) => {
+                    println!("Error: {}", err);
+                    drop(reader_s);
+                    break;
                 }
             }
-            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-                println!("Exiting...");
-                context.stop_io().await;
-                break;
-            }
-            Err(err) => {
-                println!("Error: {}", err);
-                break;
-            }
+        }
+
+        rl.save_history(".dc-history.txt")?;
+        println!("history saved");
+        Ok::<_, Error>(())
+    });
+
+    while let Ok(line) = reader_r.recv().await {
+        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
+            Ok(ExitResult::Continue) => {}
+            Ok(ExitResult::Exit) => break,
+            Err(err) => println!("Error: {}", err),
         }
     }
-
-    rl.save_history(".dc-history.txt")?;
-    println!("history saved");
+    context.stop_io().await;
+    input_loop.await?;
 
     Ok(())
 }

examples/simple.rs

@@ -1,6 +1,3 @@
-extern crate deltachat;
-
-use std::time;
 use tempfile::tempdir;
 
 use deltachat::chat;
@@ -8,35 +5,42 @@ use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
+use deltachat::message::Message;
 use deltachat::Event;
 
 fn cb(event: Event) {
-    print!("[{:?}]", event);
-
     match event {
         Event::ConfigureProgress(progress) => {
-            println!("  progress: {}", progress);
+            log::info!("progress: {}", progress);
         }
-        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            println!("  {}", msg);
+        Event::Info(msg) => {
+            log::info!("{}", msg);
         }
-        _ => {
-            println!();
+        Event::Warning(msg) => {
+            log::warn!("{}", msg);
+        }
+        Event::Error(msg) | Event::ErrorNetwork(msg) => {
+            log::error!("{}", msg);
+        }
+        event => {
+            log::info!("{:?}", event);
         }
     }
 }
 
+/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
 #[async_std::main]
 async fn main() {
+    pretty_env_logger::try_init_timed().ok();
+
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    println!("creating database {:?}", dbfile);
+    log::info!("creating database {:?}", dbfile);
     let ctx = Context::new("FakeOs".into(), dbfile.into())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
-    let duration = time::Duration::from_millis(4000);
-    println!("info: {:#?}", info);
+    log::info!("info: {:#?}", info);
 
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
@@ -45,7 +49,7 @@ async fn main() {
         }
     });
 
-    println!("configuring");
+    log::info!("configuring");
     let args = std::env::args().collect::<Vec<String>>();
     assert_eq!(args.len(), 3, "requires email password");
     let email = args[1].clone();
@@ -59,35 +63,38 @@ async fn main() {
 
     ctx.configure().await.unwrap();
 
-    println!("------ RUN ------");
-    ctx.clone().start_io().await;
-    println!("--- SENDING A MESSAGE ---");
+    log::info!("------ RUN ------");
+    ctx.start_io().await;
+    log::info!("--- SENDING A MESSAGE ---");
 
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
     let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
-    for i in 0..2 {
+    for i in 0..1 {
+        log::info!("sending message {}", i);
         chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
             .await
             .unwrap();
     }
 
-    println!("fetching chats..");
+    // wait for the message to be sent out
+    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+
+    log::info!("fetching chats..");
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let summary = chats.get_summary(&ctx, 0, None).await;
-        let text1 = summary.get_text1();
-        let text2 = summary.get_text2();
-        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
+            .await
+            .unwrap();
+        log::info!("[{}] msg: {:?}", i, msg);
     }
 
-    async_std::task::sleep(duration).await;
-
-    println!("stopping");
+    log::info!("stopping");
     ctx.stop_io().await;
-    println!("closing");
+    log::info!("closing");
+    drop(ctx);
     events_spawn.await;
 }

python/examples/echo_and_quit.py

@@ -12,7 +12,7 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
                 message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))

python/examples/test_examples.py

@@ -26,15 +26,15 @@ def test_echo_quit_plugin(acfactory, lp):
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
+    bot_chat = bot_contact.create_chat()
+    bot_chat.send_text("hello")
 
-    lp.sec("waiting for the bot-reply to arrive")
+    lp.sec("waiting for the reply message from the bot to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    assert reply.chat == ch1
     lp.sec("send quit sequence")
-    ch1.send_text("/quit")
+    bot_chat.send_text("/quit")
     botproc.wait()
 
 
@@ -47,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+    ac1.add_account_plugin(FFIEventLogger(ac1))
+    ac2.add_account_plugin(FFIEventLogger(ac2))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -60,7 +60,8 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = events.FFIEventLogger(ac, "bot")
+        ac.set_config("displayname", "bot")
+        log = events.FFIEventLogger(ac)
         ac.add_account_plugin(log)
 
     for plugin in account_plugins or []:
@@ -76,8 +77,8 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.configure()
-        ac.wait_configure_finish()
+        configtracker = ac.configure()
+        configtracker.wait_finish()
 
     # start IO threads and configure if neccessary
     ac.start_io()

python/src/deltachat/_build.py

@@ -1,65 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -71,13 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -52,11 +52,10 @@ class Account(object):
         if self._dc_context == ffi.NULL:
             raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
 
-        hook = hookspec.Global._get_plugin_manager().hook
-
         self._shutdown_event = Event()
         self._event_thread = EventThread(self)
         self._configkeys = self.get_config("sys.config_keys").split()
+        hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
     def disable_logging(self):
@@ -214,22 +213,40 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, email, name=None):
-        """ create a (new) Contact. If there already is a Contact
-        with that e-mail address, it is unblocked and its name is
-        updated.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        :param email: email-address (text type)
-        :param name: display name for this contact (optional)
+        Calling this method will always resulut in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        `name` is updated if specified.
+
+        :param obj: email-address, Account or Contact instance.
+        :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
+        if isinstance(obj, Account):
+            if not obj.is_configured():
+                raise ValueError("can only add addresses from configured accounts")
+            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
+        elif isinstance(obj, Contact):
+            if obj.account != self:
+                raise ValueError("account mismatch {}".format(obj))
+            addr, displayname = obj.addr, obj.name
+        elif isinstance(obj, str):
+            displayname, addr = parseaddr(obj)
+        else:
+            raise TypeError("don't know how to create chat for %r" % (obj, ))
+
+        if name is None and displayname:
+            name = displayname
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -251,6 +268,13 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -280,53 +304,29 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+    def create_chat(self, obj):
+        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        return self.create_contact(obj).create_chat()
 
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(contact, "id"):
-            if contact.account != self:
-                raise ValueError("Contact belongs to a different Account")
-            contact_id = contact.id
-        else:
-            assert isinstance(contact, int)
-            contact_id = contact
-        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
-        return Chat(self, chat_id)
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
 
-    def create_chat_by_message(self, message):
-        """ create or get an existing chat object for the
-        the specified message.
-
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
-        :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(message, "id"):
-            if message.account != self:
-                raise ValueError("Message belongs to a different Account")
-            msg_id = message.id
-        else:
-            assert isinstance(message, int)
-            msg_id = message
-        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
-        return Chat(self, chat_id)
-
-    def create_group_chat(self, name, verified=False):
+    def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
+        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        return Chat(self, chat_id)
+        chat = Chat(self, chat_id)
+        if contacts is not None:
+            for contact in contacts:
+                chat.add_contact(contact)
+        return chat
 
     def get_chats(self):
         """ return list of chats.
@@ -355,13 +355,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -568,29 +561,24 @@ class Account(object):
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
-        :returns: None (account is configured and with io-scheduling running)
+        :returns: None
         """
         if not self.is_configured():
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
     def configure(self):
+        """ Start configuration process and return a Configtracker instance
+        on which you can block with wait_finish() to get a True/False success
+        value for the configuration process.
+        """
         assert not self.is_configured()
-        assert not hasattr(self, "_configtracker")
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
-        if hasattr(self, "_configtracker"):
-            self.remove_account_plugin(self._configtracker)
-        self._configtracker = ConfigureTracker()
-        self.add_account_plugin(self._configtracker)
+        configtracker = ConfigureTracker(self)
+        self.add_account_plugin(configtracker)
         lib.dc_configure(self._dc_context)
-
-    def wait_configure_finish(self):
-        try:
-            self._configtracker.wait_finish()
-        finally:
-            self.remove_account_plugin(self._configtracker)
-            del self._configtracker
+        return configtracker
 
     def is_started(self):
         return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))

python/src/deltachat/chat.py

@@ -18,6 +18,8 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -137,6 +139,22 @@ class Chat(object):
         """
         return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
+    def get_ephemeral_timer(self):
+        """ get ephemeral timer.
+
+        :returns: ephemeral timer value in seconds
+        """
+        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
+
+    def set_ephemeral_timer(self, timer):
+        """ set ephemeral timer.
+
+        :param: timer value in seconds
+
+        :returns: None
+        """
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
+
     def get_type(self):
         """ (deprecated) return type of this chat.
 
@@ -328,33 +346,34 @@ class Chat(object):
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, contact):
+    def add_contact(self, obj):
         """ add a contact to this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
+        return contact
 
-    def remove_contact(self, contact):
+    def remove_contact(self, obj):
         """ remove a contact from this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
-        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-
         """
         from .contact import Contact
         dc_array = ffi.gc(
@@ -365,6 +384,14 @@ class Chat(object):
             dc_array, lambda id: Contact(self.account, id))
         )
 
+    def num_contacts(self):
+        """ return number of contacts in this chat. """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return lib.dc_array_get_cnt(dc_array)
+
     def set_profile_image(self, img_path):
         """Set group profile image.
 

python/src/deltachat/const.py

@@ -1,197 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -3,6 +3,8 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
+from .chat import Chat
+from . import const
 
 
 class Contact(object):
@@ -11,6 +13,8 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -36,10 +40,13 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def display_name(self):
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
+    # deprecated alias
+    display_name = name
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -58,6 +65,16 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)
+    def create_chat(self):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        dc_context = self.account._dc_context
+        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
+        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
+        return Chat(self.account, chat_id)
+
+    # deprecated name
+    get_chat = create_chat

python/src/deltachat/direct_imap.py

@@ -0,0 +1,213 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
+import io
+import email
+import ssl
+import pathlib
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
+
+
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+ALL = "1:*"
+
+
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account):
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        ssl_context = ssl.create_default_context()
+
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
+        try:
+            self.conn.logout()
+        except (OSError, IMAPClientError):
+            print("Could not logout direct_imap conn")
+
+    def select_folder(self, foldername):
+        assert not self._idling
+        return self.conn.select_folder(foldername)
+
+    def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
+        if "_" not in config_name:
+            config_name = "configured_{}_folder".format(config_name)
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+
+    def list_folders(self):
+        """ return list of all existing folder names"""
+        assert not self._idling
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
+
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self):
+        assert not self._idling
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
+        assert not self._idling
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.set_flags(messages, [SEEN])
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self):
+        return len(self.get_unread_messages())
+
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[HEADER]']
+                flags = data[FLAGS]
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/events.py

@@ -28,13 +28,9 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
+    def __init__(self, account):
         self.account = account
-        self.logid = logid
+        self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
@@ -127,6 +123,12 @@ class FFIEventTracker:
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
 
+    def wait_msg_delivered(self, msg):
+        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == msg.chat.id
+        assert ev.data2 == msg.id
+        assert msg.is_out_delivered()
+
 
 class EventThread(threading.Thread):
     """ Event Thread for an account.
@@ -172,7 +174,7 @@ class EventThread(threading.Thread):
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_dc_charpointer(lib.dc_event_get_data3_str(event))
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -88,6 +88,14 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
+    @global_hookspec
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -53,15 +53,19 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
+    def create_chat(self):
+        """ create or get an existing chat (group) object for this message.
+
+        If the message is a deaddrop contact request
+        the sender will become an accepted contact.
+
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
-                lib.dc_msg_unref
-        )
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):

python/src/deltachat/testplugin.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import sys
+import io
 import subprocess
 import queue
 import threading
@@ -15,8 +16,8 @@ import requests
 from . import Account, const
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
-from _pytest.monkeypatch import MonkeyPatch
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
@@ -34,9 +35,6 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
@@ -101,17 +99,16 @@ def pytest_report_header(config, startdir):
     summary = []
 
     t = tempfile.mktemp()
-    m = MonkeyPatch()
     try:
         ac = Account(t)
         info = ac.get_info()
         ac.shutdown()
     finally:
-        m.undo()
         os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
+    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
          info['deltachat_core_version'],
          info['sqlite_version'],
+         info['journal_mode'],
      )])
 
     cfg = config.option.liveconfig
@@ -218,6 +215,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -228,12 +226,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+                ac.add_account_plugin(FFIEventLogger(ac))
             self._accounts.append(ac)
             return ac
 
@@ -302,25 +303,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1.wait_configure_finish()
-            ac1.start_io()
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1.wait_configure_finish()
-            ac1.start_io()
-            ac2.wait_configure_finish()
-            ac2.start_io()
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+                        for i in range(num)]
+            self.wait_configure_and_start_io()
+            for acc in accounts:
+                acc.add_account_plugin(FFIEventLogger(acc))
+            return accounts
+
         def clone_online_account(self, account, pre_generated_key=True):
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
@@ -336,14 +341,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
                 "-u",
@@ -368,9 +388,42 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    return am
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
@@ -438,4 +491,17 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
+
     return Printer()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # set a report attribute for each phase of a call, which can
+    # be "setup", "call", "teardown"
+
+    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -40,12 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -59,7 +61,10 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
+        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """

python/tests/stress_test_db.py

@@ -0,0 +1,135 @@
+
+import time
+import threading
+import pytest
+import os
+from queue import Queue, Empty
+
+import deltachat
+
+
+def test_db_busy_error(acfactory, tmpdir):
+    starttime = time.time()
+    log_lock = threading.RLock()
+
+    def log(string):
+        with log_lock:
+            print("%3.2f %s" % (time.time() - starttime, string))
+
+    # make a number of accounts
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
+    log("created %s accounts" % len(accounts))
+
+    # put a bigfile into each account
+    for acc in accounts:
+        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
+        with open(acc.bigfile, "wb") as f:
+            f.write(b"01234567890"*1000_000)
+    log("created %s bigfiles" % len(accounts))
+
+    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
+    chat = accounts[0].create_group_chat("stress-group")
+    for addr in contact_addrs[1:]:
+        chat.add_contact(chat.account.create_contact(addr))
+
+    # setup auto-responder bots which report back failures/actions
+    report_queue = Queue()
+
+    def report_func(replier, report_type, *report_args):
+        report_queue.put((replier, report_type, report_args))
+
+    # each replier receives all events and sends report events to receive_queue
+    repliers = []
+    for acc in accounts:
+        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
+        acc.add_account_plugin(replier)
+        repliers.append(replier)
+
+    # kick off message sending
+    # after which repliers will reply to each other
+    chat.send_text("hello")
+
+    alive_count = len(accounts)
+    while alive_count > 0:
+        try:
+            replier, report_type, report_args = report_queue.get(timeout=10)
+        except Empty:
+            log("timeout waiting for next event")
+            pytest.fail("timeout exceeded")
+        if report_type == ReportType.exit:
+            replier.log("EXIT")
+        elif report_type == ReportType.ffi_error:
+            replier.log("ERROR: {}".format(report_args[0]))
+        elif report_type == ReportType.message_echo:
+            continue
+        else:
+            raise ValueError("{} unknown report type {}, args={}".format(
+                addr, report_type, report_args
+            ))
+        alive_count -= 1
+        replier.log("shutting down")
+        replier.account.shutdown()
+        replier.log("shut down complete, remaining={}".format(alive_count))
+
+
+class ReportType:
+    exit = "exit"
+    ffi_error = "ffi-error"
+    message_echo = "message-echo"
+
+
+class AutoReplier:
+    def __init__(self, account, log, num_send, num_bigfiles, report_func):
+        self.account = account
+        self._log = log
+        self.report_func = report_func
+        self.num_send = num_send
+        self.num_bigfiles = num_bigfiles
+        self.current_sent = 0
+        self.addr = self.account.get_self_contact().addr
+
+        self._thread = threading.Thread(
+            name="Stats{}".format(self.account),
+            target=self.thread_stats
+        )
+        self._thread.setDaemon(True)
+        self._thread.start()
+
+    def log(self, message):
+        self._log("{} {}".format(self.addr, message))
+
+    def thread_stats(self):
+        # XXX later use, for now we just quit
+        return
+        while 1:
+            time.sleep(1.0)
+            break
+
+    @deltachat.account_hookimpl
+    def ac_incoming_message(self, message):
+        if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
+            return
+        message.create_chat()
+        message.mark_seen()
+        self.log("incoming message: {}".format(message))
+
+        self.current_sent += 1
+        # we are still alive, let's send a reply
+        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
+            message.chat.send_text("send big file as reply to: {}".format(message.text))
+            msg = message.chat.send_file(self.account.bigfile)
+        else:
+            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
+            assert msg.text
+        self.log("message-sent: {}".format(msg))
+        self.report_func(self, ReportType.message_echo)
+        if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
+            return
+
+    @deltachat.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self.log(ffi_event)
+        if ffi_event.name == "DC_EVENT_ERROR":
+            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_increation.py

@@ -36,9 +36,7 @@ def wait_msgs_changed(account, msgs_list):
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -48,9 +46,7 @@ class TestOnlineInCreation:
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -64,9 +60,7 @@ class TestOnlineInCreation:
     def test_forward_increation(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
@@ -80,7 +74,7 @@ class TestOnlineInCreation:
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(c2)
+        chat2.add_contact(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them

python/tests/test_lowlevel.py

@@ -69,8 +69,8 @@ def test_sig():
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
 
-    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    chat = ac1.create_chat_by_contact(contact1)
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]

python/tox.ini

@@ -71,6 +71,8 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
+markers = 
+    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

spec.md

@@ -1,10 +1,12 @@
-# Chat-over-Email specification
+# chat-mail specification
 
-Version 0.30.0
+Version: 0.32.0
+Status:  In-progress 
+Format:  [Semantic Line Breaks](https://sembr.org/)
 
-This document describes how emails can be used
-to implement typical messenger functions
-while staying compatible to existing MUAs.
+This document roughly describes how chat-mail 
+apps use the standard e-mail system 
+to implement typical messenger functions.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -30,17 +32,14 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
-If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD start with the characters `Chat:`
-and SHOULD be an excerpt of the message.
+the `Subject` header SHOULD be `Message from <sender name>`.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -58,7 +57,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Chat: Hello ...
+    Subject: Message from sender@domain
 
     Hello world!
 

src/blob.rs

@@ -8,11 +8,15 @@ use async_std::prelude::*;
 use async_std::{fs, io};
 
 use image::GenericImageView;
+use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::constants::AVATAR_SIZE;
+use crate::config::Config;
+use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::Event;
+use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -57,7 +61,7 @@ impl<'a> BlobObject<'a> {
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -372,11 +376,73 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
     }
+
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
+            != Some((Viewtype::Image, "image/jpeg"))
+        {
+            return Ok(());
+        }
+
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
+
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
+        }
+
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
+    }
+
+    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exifreader = exif::Reader::new();
+        let exif = exifreader.read_from_container(&mut bufreader)?;
+        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
+            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
+            // we only use rotation, in practise, flipping is not used.
+            match orientation.value.get_uint(0) {
+                Some(3) => return Ok(180),
+                Some(6) => return Ok(90),
+                Some(8) => return Ok(270),
+                other => warn!(context, "exif orientation value ignored: {:?}", other),
+            }
+        }
+        Ok(0)
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -400,7 +466,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
@@ -431,7 +497,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
         let fname = t.ctx.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
@@ -442,7 +508,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_lowercase_ext() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
             .await
             .unwrap();
@@ -451,7 +517,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_file_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -460,7 +526,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_rel_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -469,7 +535,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_suffix() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -480,7 +546,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_dup() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -504,7 +570,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_double_ext_preserved() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
             .await
             .unwrap();
@@ -529,7 +595,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_long_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let s = "1".repeat(150);
         let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
@@ -538,7 +604,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_and_copy() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
         let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
@@ -554,7 +620,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_from_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
@@ -572,7 +638,7 @@ mod tests {
     }
     #[async_std::test]
     async fn test_create_from_name_long() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
         let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
@@ -594,8 +660,40 @@ mod tests {
 
     #[test]
     fn test_sanitise_name() {
-        let (_, ext) =
+        let (stem, ext) =
             BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
         assert_eq!(ext, ".txt");
+        assert!(!stem.is_empty());
+
+        // the extensions are kept together as between stem and extension a number may be added -
+        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
+        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
+        assert_eq!(stem, "wot");
+        assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
+        assert_eq!(stem, "");
+        assert_eq!(ext, ".foo.bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
+        assert!(stem.contains("foo"));
+        assert!(!stem.contains("?"));
+        assert_eq!(ext, ".bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("no-extension");
+        assert_eq!(stem, "no-extension");
+        assert_eq!(ext, "");
+
+        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
+        assert_eq!(ext, ".c");
+        assert!(!stem.contains("path"));
+        assert!(!stem.contains("ignored"));
+        assert!(stem.contains("this"));
+        assert!(stem.contains("forbidden"));
+        assert!(!stem.contains("/"));
+        assert!(!stem.contains("\\"));
+        assert!(!stem.contains(":"));
+        assert!(!stem.contains("*"));
+        assert!(!stem.contains("?"));
     }
 }

src/chat.rs

@@ -15,6 +15,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::{delete_expired_messages, schedule_ephemeral_task, Timer as EphemeralTimer};
 use crate::error::{bail, ensure, format_err, Error};
 use crate::events::Event;
 use crate::job::{self, Action};
@@ -24,6 +25,25 @@ use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
+/// An chat item, such as a message or a marker.
+#[derive(Debug, Copy, Clone)]
+pub enum ChatItem {
+    Message {
+        msg_id: MsgId,
+    },
+
+    /// A marker without inherent meaning. It is inserted before user
+    /// supplied MsgId.
+    Marker1,
+
+    /// Day marker, separating messages that correspond to different
+    /// days according to local time.
+    DayMarker {
+        /// Marker timestamp, for day markers
+        timestamp: i64,
+    },
+}
+
 /// Chat ID, including reserved IDs.
 ///
 /// Some chat IDs are reserved to identify special chat types.  This
@@ -39,18 +59,9 @@ impl ChatId {
         ChatId(id)
     }
 
-    /// A ChatID which indicates an error.
-    ///
-    /// This is transitional and should not be used in new code.  Do
-    /// not represent errors in a ChatId.
-    pub fn is_error(self) -> bool {
-        self.0 == 0
-    }
-
     /// An unset ChatId
     ///
-    /// Like [ChatId::is_error], from which it is indistinguishable, this is
-    /// transitional and should not be used in new code.
+    /// This is transitional and should not be used in new code.
     pub fn is_unset(self) -> bool {
         self.0 == 0
     }
@@ -431,24 +442,44 @@ impl ChatId {
     }
 
     async fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
-        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
-        self.parent_query(
-            context,
-            "rfc724_mid, mime_in_reply_to, mime_references",
-            collect,
-        )
-        .await
-        .ok()
-        .flatten()
+        let collect =
+            |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?));
+        let (rfc724_mid, mime_in_reply_to, mime_references, error): (
+            String,
+            String,
+            String,
+            String,
+        ) = self
+            .parent_query(
+                context,
+                "rfc724_mid, mime_in_reply_to, mime_references, error",
+                collect,
+            )
+            .await
+            .ok()
+            .flatten()?;
+
+        if !error.is_empty() {
+            // Do not reply to error messages.
+            //
+            // An error message could be a group chat message that we failed to decrypt and
+            // assigned to 1:1 chat. A reply to it will show up as a reply to group message
+            // on the other side. To avoid such situations, it is better not to reply to
+            // error messages at all.
+            None
+        } else {
+            Some((rfc724_mid, mime_in_reply_to, mime_references))
+        }
     }
 
     async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
-        let collect = |row: &rusqlite::Row| Ok(row.get(0)?);
-        let packed: Option<String> = self.parent_query(context, "param", collect).await?;
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?));
+        let res: Option<(String, String)> =
+            self.parent_query(context, "param, error", collect).await?;
 
-        if let Some(ref packed) = packed {
+        if let Some((ref packed, ref error)) = res {
             let param = packed.parse::<Params>()?;
-            Ok(param.exists(Param::GuaranteeE2ee))
+            Ok(error.is_empty() && param.exists(Param::GuaranteeE2ee))
         } else {
             // No messages
             Ok(false)
@@ -699,6 +730,7 @@ impl Chat {
                 .unwrap_or_else(std::path::PathBuf::new),
             draft,
             is_muted: self.is_muted(),
+            ephemeral_timer: self.id.get_ephemeral_timer(context).await?,
         })
     }
 
@@ -927,10 +959,20 @@ impl Chat {
                     .await?;
             }
 
+            let ephemeral_timer = if msg.param.get_cmd() == SystemMessage::EphemeralTimerChanged {
+                EphemeralTimer::Disabled
+            } else {
+                self.id.get_ephemeral_timer(context).await?
+            };
+            let ephemeral_timestamp = match ephemeral_timer {
+                EphemeralTimer::Disabled => 0,
+                EphemeralTimer::Enabled { duration } => timestamp + i64::from(duration),
+            };
+
             // add message to the database
 
             if context.sql.execute(
-                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?);",
+                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id, ephemeral_timer, ephemeral_timestamp) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?);",
                         paramsv![
                             new_rfc724_mid,
                             self.id,
@@ -945,6 +987,8 @@ impl Chat {
                             new_in_reply_to,
                             new_references,
                             location_id as i32,
+                            ephemeral_timer,
+                            ephemeral_timestamp
                         ]
                     ).await.is_ok() {
                         msg_id = context.sql.get_rowid(
@@ -963,6 +1007,7 @@ impl Chat {
         } else {
             error!(context, "Cannot send message, not configured.",);
         }
+        schedule_ephemeral_task(context).await;
 
         Ok(MsgId::new(msg_id))
     }
@@ -990,13 +1035,13 @@ impl rusqlite::types::ToSql for ChatVisibility {
 
 impl rusqlite::types::FromSql for ChatVisibility {
     fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-        i64::column_result(value).and_then(|val| {
+        i64::column_result(value).map(|val| {
             match val {
-                2 => Ok(ChatVisibility::Pinned),
-                1 => Ok(ChatVisibility::Archived),
-                0 => Ok(ChatVisibility::Normal),
+                2 => ChatVisibility::Pinned,
+                1 => ChatVisibility::Archived,
+                0 => ChatVisibility::Normal,
                 // fallback to to Normal for unknown values, may happen eg. on imports created by a newer version.
-                _ => Ok(ChatVisibility::Normal),
+                _ => ChatVisibility::Normal,
             }
         })
     }
@@ -1059,6 +1104,9 @@ pub struct ChatInfo {
     ///
     /// The exact time its muted can be found out via the `chat.mute_duration` property
     pub is_muted: bool,
+
+    /// Ephemeral message timer.
+    pub ephemeral_timer: EphemeralTimer,
     // ToDo:
     // - [ ] deaddrop,
     // - [ ] summary,
@@ -1341,6 +1389,12 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Er
             .ok_or_else(|| {
                 format_err!("Attachment missing for message of type #{}", msg.viewtype)
             })?;
+
+        if msg.viewtype == Viewtype::Image {
+            if let Err(e) = blob.recode_to_image_size(context).await {
+                warn!(context, "Cannot recode image, using original data: {:?}", e);
+            }
+        }
         msg.param.set(Param::File, blob.as_name());
 
         if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
@@ -1447,7 +1501,7 @@ pub async fn send_msg(
                 }
             }
             msg.param.remove(Param::PrepForwards);
-            msg.save_param_to_disk(context).await;
+            msg.update_param(context).await;
         }
         return send_msg_inner(context, chat_id, msg).await;
     }
@@ -1563,11 +1617,16 @@ pub async fn get_chat_msgs(
     chat_id: ChatId,
     flags: u32,
     marker1before: Option<MsgId>,
-) -> Vec<MsgId> {
-    match delete_device_expired_messages(context).await {
+) -> Vec<ChatItem> {
+    match delete_expired_messages(context).await {
         Err(err) => warn!(context, "Failed to delete expired messages: {}", err),
         Ok(messages_deleted) => {
             if messages_deleted {
+                // Trigger reload of chatlist.
+                //
+                // On desktop chatlist is always shown on the side,
+                // and it is important to update the last message shown
+                // there.
                 context.emit_event(Event::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
@@ -1586,18 +1645,20 @@ pub async fn get_chat_msgs(
             let (curr_id, ts) = row?;
             if let Some(marker_id) = marker1before {
                 if curr_id == marker_id {
-                    ret.push(MsgId::new(DC_MSG_ID_MARKER1));
+                    ret.push(ChatItem::Marker1);
                 }
             }
             if (flags & DC_GCM_ADDDAYMARKER) != 0 {
                 let curr_local_timestamp = ts + cnv_to_local;
                 let curr_day = curr_local_timestamp / 86400;
                 if curr_day != last_day {
-                    ret.push(MsgId::new(DC_MSG_ID_DAYMARKER));
+                    ret.push(ChatItem::DayMarker {
+                        timestamp: curr_day,
+                    });
                     last_day = curr_day;
                 }
             }
-            ret.push(curr_id);
+            ret.push(ChatItem::Message { msg_id: curr_id });
         }
         Ok(ret)
     };
@@ -1729,52 +1790,6 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-/// Deletes messages which are expired according to "delete_device_after" setting.
-///
-/// Returns true if any message is deleted, so event can be emitted. If nothing
-/// has been deleted, returns false.
-pub async fn delete_device_expired_messages(context: &Context) -> Result<bool, Error> {
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
-        let threshold_timestamp = time() - delete_device_after;
-
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .await
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .await
-            .unwrap_or_default()
-            .0;
-
-        // Delete expired messages
-        //
-        // Only update the rows that have to be updated, to avoid emitting
-        // unnecessary "chat modified" events.
-        let rows_modified = context
-            .sql
-            .execute(
-                "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
-             WHERE timestamp < ? \
-             AND chat_id > ? \
-             AND chat_id != ? \
-             AND chat_id != ?",
-                paramsv![
-                    DC_CHAT_ID_TRASH,
-                    threshold_timestamp,
-                    DC_CHAT_ID_LAST_SPECIAL,
-                    self_chat_id,
-                    device_chat_id
-                ],
-            )
-            .await?;
-
-        Ok(rows_modified > 0)
-    } else {
-        Ok(false)
-    }
-}
-
 pub async fn get_chat_media(
     context: &Context,
     chat_id: ChatId,
@@ -1930,20 +1945,19 @@ pub async fn create_group_chat(
         .sql
         .get_rowid(context, "chats", "grpid", grpid)
         .await?;
-    let chat_id = ChatId::new(row_id);
-    if !chat_id.is_error() {
-        if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
-            let mut draft_msg = Message::new(Viewtype::Text);
-            draft_msg.set_text(Some(draft_txt));
-            chat_id.set_draft_raw(context, &mut draft_msg).await;
-        }
 
-        context.emit_event(Event::MsgsChanged {
-            msg_id: MsgId::new(0),
-            chat_id: ChatId::new(0),
-        });
+    let chat_id = ChatId::new(row_id);
+    if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
+        let mut draft_msg = Message::new(Viewtype::Text);
+        draft_msg.set_text(Some(draft_txt));
+        chat_id.set_draft_raw(context, &mut draft_msg).await;
     }
 
+    context.emit_event(Event::MsgsChanged {
+        msg_id: MsgId::new(0),
+        chat_id: ChatId::new(0),
+    });
+
     Ok(chat_id)
 }
 
@@ -2580,7 +2594,7 @@ pub async fn forward_msgs(
                         .set(Param::PrepForwards, new_msg_id.to_u32().to_string());
                 }
 
-                msg.save_param_to_disk(context).await;
+                msg.update_param(context).await;
                 msg.param = save_param;
             } else {
                 msg.state = MessageState::OutPending;
@@ -2767,9 +2781,16 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
 /// For example, it can be a message showing that a member was added to a group.
 pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
+    let ephemeral_timer = match chat_id.get_ephemeral_timer(context).await {
+        Err(e) => {
+            warn!(context, "Could not get timer for info msg: {}", e);
+            return;
+        }
+        Ok(ephemeral_timer) => ephemeral_timer,
+    };
 
-    if context.sql.execute(
-        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid) VALUES (?,?,?, ?,?,?, ?,?);",
+    if let Err(e) = context.sql.execute(
+        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid,ephemeral_timer) VALUES (?,?,?, ?,?,?, ?,?,?);",
         paramsv![
             chat_id,
             DC_CONTACT_ID_INFO,
@@ -2779,8 +2800,10 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
             MessageState::InNoticed,
             text.as_ref().to_string(),
             rfc724_mid,
+            ephemeral_timer
         ]
-    ).await.is_err() {
+    ).await {
+        warn!(context, "Could not add info msg: {}", e);
         return;
     }
 
@@ -2804,7 +2827,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_chat_info() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let bob = Contact::create(&t.ctx, "bob", "bob@example.com")
             .await
             .unwrap();
@@ -2827,7 +2850,8 @@ mod tests {
                 "color": 15895624,
                 "profile_image": "",
                 "draft": "",
-                "is_muted": false
+                "is_muted": false,
+                "ephemeral_timer": "Disabled"
             }
         "#;
 
@@ -2838,7 +2862,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_draft_no_draft() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2848,7 +2872,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_draft_special_chat_id() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let draft = ChatId::new(DC_CHAT_ID_LAST_SPECIAL)
             .get_draft(&t.ctx)
             .await
@@ -2860,14 +2884,14 @@ mod tests {
     async fn test_get_draft_no_chat() {
         // This is a weird case, maybe this should be an error but we
         // do not get this info from the database currently.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let draft = ChatId::new(42).get_draft(&t.ctx).await.unwrap();
         assert!(draft.is_none());
     }
 
     #[async_std::test]
     async fn test_get_draft() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2883,7 +2907,7 @@ mod tests {
     #[async_std::test]
     async fn test_add_contact_to_chat_ex_add_self() {
         // Adding self to a contact should succeed, even though it's pointless.
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -2895,7 +2919,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_self_talk() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2916,7 +2940,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_deaddrop_chat() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat = Chat::load_from_db(&t.ctx, ChatId::new(DC_CHAT_ID_DEADDROP))
             .await
             .unwrap();
@@ -2931,7 +2955,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_unlabelled() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         // add two device-messages
         let mut msg1 = Message::new(Viewtype::Text);
@@ -2966,7 +2990,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_labelled() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         // add two device-messages with the same label (second attempt is not added)
         let mut msg1 = Message::new(Viewtype::Text);
@@ -3020,7 +3044,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_label_only() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let res = add_device_msg(&t.ctx, Some(""), None).await;
         assert!(res.is_err());
         let res = add_device_msg(&t.ctx, Some("some-label"), None).await;
@@ -3040,7 +3064,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_was_device_msg_ever_added() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         add_device_msg(&t.ctx, Some("some-label"), None).await.ok();
         assert!(was_device_msg_ever_added(&t.ctx, "some-label")
             .await
@@ -3064,7 +3088,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_delete_device_chat() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("message text".to_string());
@@ -3084,7 +3108,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_device_chat_cannot_sent() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
         let (device_chat_id, _) =
             create_or_lookup_by_contact_id(&t.ctx, DC_CONTACT_ID_DEVICE, Blocked::Not)
@@ -3104,7 +3128,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_delete_and_reset_all_device_msgs() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("message text".to_string());
         let msg_id1 = add_device_msg(&t.ctx, Some("some-label"), Some(&mut msg))
@@ -3141,7 +3165,7 @@ mod tests {
     #[async_std::test]
     async fn test_archive() {
         // create two chats
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("foo".to_string());
         let msg_id = add_device_msg(&t.ctx, None, Some(&mut msg)).await.unwrap();
@@ -3255,7 +3279,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_pinned() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // create 3 chats, wait 1 second in between to get a reliable order (we order by time)
         let mut msg = Message::new(Viewtype::Text);
@@ -3314,7 +3338,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_chat_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3338,7 +3362,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_same_chat_twice() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let contact1 = Contact::create(&context.ctx, "bob", "bob@mail.de")
             .await
             .unwrap();
@@ -3357,7 +3381,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_shall_attach_selfavatar() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3381,7 +3405,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_mute_duration() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3449,7 +3473,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_parent_is_encrypted() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();

src/chatlist.rs

@@ -5,6 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
+use crate::ephemeral::delete_expired_messages;
 use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
@@ -76,7 +77,7 @@ impl Chatlist {
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
     ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+    ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -99,7 +100,7 @@ impl Chatlist {
 
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context).await {
+        if let Err(err) = delete_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -147,11 +148,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
@@ -173,11 +175,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
@@ -206,11 +209,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND c.name LIKE ?3
@@ -236,11 +240,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND NOT c.archived=?3
@@ -424,7 +429,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_try_load() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
@@ -472,7 +477,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
         create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
@@ -497,7 +502,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_search_special_chat_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
 
         let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
@@ -530,7 +535,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_summary_unwrap() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();

src/config.rs

@@ -104,6 +104,9 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
+    ConfiguredInboxFolder,
+    ConfiguredMvboxFolder,
+    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -117,6 +120,10 @@ pub enum Config {
 }
 
 impl Context {
+    pub async fn config_exists(&self, key: Config) -> bool {
+        self.sql.get_raw_config(self, key).await.is_some()
+    }
+
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
     pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
@@ -137,6 +144,7 @@ impl Context {
         // Default values
         match key {
             Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
@@ -197,21 +205,6 @@ impl Context {
                     None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
-            Config::InboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_inbox().await;
-                ret
-            }
-            Config::SentboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_sentbox().await;
-                ret
-            }
-            Config::MvboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_mvbox().await;
-                ret
-            }
             Config::Selfstatus => {
                 let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
@@ -281,7 +274,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_outside_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         File::create(&avatar_src)
@@ -310,7 +303,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_in_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.ctx.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
@@ -336,7 +329,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
         File::create(&avatar_src)
@@ -360,7 +353,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_media_quality_config_option() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure/mod.rs

@@ -1,10 +1,12 @@
 //! Email accounts autoconfiguration process module
 
+#![forbid(clippy::indexing_slicing)]
+
 mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
 
-use anyhow::{bail, ensure, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use async_std::prelude::*;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
@@ -66,65 +68,22 @@ impl Context {
     }
 
     async fn inner_configure(&self) -> Result<()> {
-        let mut success = false;
-        let mut param_autoconfig: Option<LoginParam> = None;
-
         info!(self, "Configure ...");
 
-        // Variables that are shared between steps:
         let mut param = LoginParam::from_database(self, "").await;
-        // need all vars here to be mutable because rust thinks the same step could be called multiple times
-        // and also initialize, because otherwise rust thinks it's used while unitilized, even if thats not the case as the loop goes only forward
-        let mut param_domain = "undefined.undefined".to_owned();
-        let mut param_addr_urlencoded: String =
-            "Internal Error: this value should never be used".to_owned();
-        let mut keep_flags = 0;
-
-        let mut step_counter: u8 = 0;
-        let (_s, r) = async_std::sync::channel(1);
-        let mut imap = Imap::new(r);
-        let was_configured_before = self.is_configured().await;
-
-        while !self.shall_stop_ongoing().await {
-            step_counter += 1;
-
-            match exec_step(
-                self,
-                &mut imap,
-                &mut param,
-                &mut param_domain,
-                &mut param_autoconfig,
-                &mut param_addr_urlencoded,
-                &mut keep_flags,
-                &mut step_counter,
-            )
-            .await
-            {
-                Ok(step) => {
-                    success = true;
-                    match step {
-                        Step::Continue => {}
-                        Step::Done => break,
-                    }
-                }
-                Err(err) => {
-                    error!(self, "{}", err);
-                    success = false;
-                    break;
-                }
-            }
-        }
-
-        if imap.is_connected() {
-            imap.disconnect(self).await;
-        }
+        let success = configure(self, &mut param).await;
 
         if let Some(provider) = provider::get_provider_info(&param.addr) {
-            if !was_configured_before {
-                if let Some(config_defaults) = &provider.config_defaults {
-                    for def in config_defaults.iter() {
+            if let Some(config_defaults) = &provider.config_defaults {
+                for def in config_defaults.iter() {
+                    if !self.config_exists(def.key).await {
                         info!(self, "apply config_defaults {}={}", def.key, def.value);
                         self.set_config(def.key, Some(def.value)).await?;
+                    } else {
+                        info!(
+                            self,
+                            "skip already set config_defaults {}={}", def.key, def.value
+                        );
                     }
                 }
             }
@@ -141,329 +100,294 @@ impl Context {
             }
         }
 
-        // remember the entered parameters on success
-        // and restore to last-entered on failure.
-        // this way, the parameters visible to the ui are always in-sync with the current configuration.
-        if success {
-            LoginParam::from_database(self, "")
-                .await
-                .save_to_database(self, "configured_raw_")
-                .await
-                .ok();
+        match success {
+            Ok(_) => {
+                progress!(self, 1000);
+                Ok(())
+            }
+            Err(err) => {
+                error!(self, "Configure Failed: {}", err);
+                progress!(self, 0);
+                Err(err)
+            }
+        }
+    }
+}
 
-            progress!(self, 1000);
-            Ok(())
+async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
+    let mut param_autoconfig: Option<LoginParam> = None;
+    let mut keep_flags = 0;
+
+    // Read login parameters from the database
+    progress!(ctx, 1);
+    ensure!(!param.addr.is_empty(), "Please enter an email address.");
+
+    // Step 1: Load the parameters and check email-address and password
+
+    if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
+        // the used oauth2 addr may differ, check this.
+        // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
+        progress!(ctx, 10);
+        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.mail_pw)
+            .await
+            .and_then(|e| e.parse().ok())
+        {
+            info!(ctx, "Authorized address is {}", oauth2_addr);
+            param.addr = oauth2_addr;
+            ctx.sql
+                .set_raw_config(ctx, "addr", Some(param.addr.as_str()))
+                .await?;
+        }
+        progress!(ctx, 20);
+    }
+    // no oauth? - just continue it's no error
+
+    let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
+    let param_domain = parsed.domain;
+    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
+
+    // Step 2: Autoconfig
+    progress!(ctx, 200);
+
+    // param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then
+    // param.send_pw.is_empty()   -- the password cannot be auto-configured and is no criterion for
+    //                               autoconfig or not
+    if param.mail_server.is_empty()
+        && param.mail_port == 0
+        && param.send_server.is_empty()
+        && param.send_port == 0
+        && param.send_user.is_empty()
+        && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
+    {
+        // no advanced parameters entered by the user: query provider-database or do Autoconfig
+        keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
+        if let Some(new_param) = get_offline_autoconfig(ctx, &param) {
+            // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
+            param_autoconfig = Some(new_param);
+        }
+
+        if param_autoconfig.is_none() {
+            param_autoconfig =
+                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
+        }
+    }
+
+    // C.  Do we have any autoconfig result?
+    progress!(ctx, 500);
+    if let Some(ref cfg) = param_autoconfig {
+        info!(ctx, "Got autoconfig: {}", &cfg);
+        if !cfg.mail_user.is_empty() {
+            param.mail_user = cfg.mail_user.clone();
+        }
+        // all other values are always NULL when entering autoconfig
+        param.mail_server = cfg.mail_server.clone();
+        param.mail_port = cfg.mail_port;
+        param.send_server = cfg.send_server.clone();
+        param.send_port = cfg.send_port;
+        param.send_user = cfg.send_user.clone();
+        param.server_flags = cfg.server_flags;
+        // although param_autoconfig's data are no longer needed from,
+        // it is used to later to prevent trying variations of port/server/logins
+    }
+    param.server_flags |= keep_flags;
+
+    // Step 3: Fill missing fields with defaults
+    if param.mail_server.is_empty() {
+        param.mail_server = format!("imap.{}", param_domain,)
+    }
+    if param.mail_port == 0 {
+        param.mail_port = if 0 != param.server_flags & (0x100 | 0x400) {
+            143
         } else {
-            LoginParam::from_database(self, "configured_raw_")
-                .await
-                .save_to_database(self, "")
-                .await
-                .ok();
-
-            progress!(self, 0);
-            bail!("Configure failed")
+            993
         }
     }
+    if param.mail_user.is_empty() {
+        param.mail_user = param.addr.clone();
+    }
+    if param.send_server.is_empty() && !param.mail_server.is_empty() {
+        param.send_server = param.mail_server.clone();
+        if param.send_server.starts_with("imap.") {
+            param.send_server = param.send_server.replacen("imap", "smtp", 1);
+        }
+    }
+    if param.send_port == 0 {
+        param.send_port = if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
+            587
+        } else if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
+            25
+        } else {
+            465
+        }
+    }
+    if param.send_user.is_empty() && !param.mail_user.is_empty() {
+        param.send_user = param.mail_user.clone();
+    }
+    if param.send_pw.is_empty() && !param.mail_pw.is_empty() {
+        param.send_pw = param.mail_pw.clone()
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
+        param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
+        param.server_flags |= DC_LP_AUTH_NORMAL as i32
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= if param.send_port == 143 {
+            DC_LP_IMAP_SOCKET_STARTTLS as i32
+        } else {
+            DC_LP_IMAP_SOCKET_SSL as i32
+        }
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= if param.send_port == 587 {
+            DC_LP_SMTP_SOCKET_STARTTLS as i32
+        } else if param.send_port == 25 {
+            DC_LP_SMTP_SOCKET_PLAIN as i32
+        } else {
+            DC_LP_SMTP_SOCKET_SSL as i32
+        }
+    }
+
+    // do we have a complete configuration?
+    ensure!(
+        !param.mail_server.is_empty()
+            && param.mail_port != 0
+            && !param.mail_user.is_empty()
+            && !param.mail_pw.is_empty()
+            && !param.send_server.is_empty()
+            && param.send_port != 0
+            && !param.send_user.is_empty()
+            && !param.send_pw.is_empty()
+            && param.server_flags != 0,
+        "Account settings incomplete."
+    );
+
+    progress!(ctx, 600);
+    // try to connect to IMAP - if we did not got an autoconfig,
+    // do some further tries with different settings and username variations
+    let (_s, r) = async_std::sync::channel(1);
+    let mut imap = Imap::new(r);
+
+    try_imap_connections(ctx, param, param_autoconfig.is_some(), &mut imap).await?;
+    progress!(ctx, 800);
+
+    try_smtp_connections(ctx, param, param_autoconfig.is_some()).await?;
+    progress!(ctx, 900);
+
+    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
+        || ctx.get_config_bool(Config::MvboxMove).await;
+
+    imap.configure_folders(ctx, create_mvbox).await?;
+
+    imap.select_with_uidvalidity(ctx, "INBOX")
+        .await
+        .context("could not read INBOX status")?;
+
+    drop(imap);
+
+    progress!(ctx, 910);
+    // configuration success - write back the configured parameters with the
+    // "configured_" prefix; also write the "configured"-flag */
+    // the trailing underscore is correct
+    param.save_to_database(ctx, "configured_").await?;
+    ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
+
+    progress!(ctx, 920);
+
+    e2ee::ensure_secret_key_exists(ctx).await?;
+    info!(ctx, "key generation completed");
+
+    progress!(ctx, 940);
+
+    Ok(())
 }
 
-#[allow(clippy::too_many_arguments)]
-async fn exec_step(
-    ctx: &Context,
-    imap: &mut Imap,
-    param: &mut LoginParam,
-    param_domain: &mut String,
-    param_autoconfig: &mut Option<LoginParam>,
-    param_addr_urlencoded: &mut String,
-    keep_flags: &mut i32,
-    step_counter: &mut u8,
-) -> Result<Step> {
-    const STEP_12_USE_AUTOCONFIG: u8 = 12;
-    const STEP_13_AFTER_AUTOCONFIG: u8 = 13;
+#[derive(Debug, PartialEq, Eq)]
+enum AutoconfigProvider {
+    Mozilla,
+    Outlook,
+}
 
-    match *step_counter {
-        // Read login parameters from the database
-        1 => {
-            progress!(ctx, 1);
-            ensure!(!param.addr.is_empty(), "Please enter an email address.");
-        }
-        // Step 1: Load the parameters and check email-address and password
-        2 => {
-            if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
-                // the used oauth2 addr may differ, check this.
-                // if dc_get_oauth2_addr() is not available in the oauth2 implementation,
-                // just use the given one.
-                progress!(ctx, 10);
-                if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.mail_pw)
-                    .await
-                    .and_then(|e| e.parse().ok())
-                {
-                    info!(ctx, "Authorized address is {}", oauth2_addr);
-                    param.addr = oauth2_addr;
-                    ctx.sql
-                        .set_raw_config(ctx, "addr", Some(param.addr.as_str()))
-                        .await?;
-                }
-                progress!(ctx, 20);
-            }
-            // no oauth? - just continue it's no error
-        }
-        3 => {
-            if let Ok(parsed) = param.addr.parse() {
-                let parsed: EmailAddress = parsed;
-                *param_domain = parsed.domain;
-                *param_addr_urlencoded =
-                    utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
-            } else {
-                bail!("Bad email-address.");
-            }
-        }
-        // Step 2: Autoconfig
-        4 => {
-            progress!(ctx, 200);
+#[derive(Debug, PartialEq, Eq)]
+struct AutoconfigSource {
+    provider: AutoconfigProvider,
+    url: String,
+}
 
-            if param.mail_server.is_empty()
-                && param.mail_port == 0
-            /* && param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then */
-                && param.send_server.is_empty()
-                && param.send_port == 0
-                && param.send_user.is_empty()
-            /* && param.send_pw.is_empty() -- the password cannot be auto-configured and is no criterion for autoconfig or not */
-                && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
-            {
-                // no advanced parameters entered by the user: query provider-database or do Autoconfig
-                *keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
-                if let Some(new_param) = get_offline_autoconfig(ctx, &param) {
-                    // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
-                    *param_autoconfig = Some(new_param);
-                    *step_counter = STEP_12_USE_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
-                }
-            } else {
-                // advanced parameters entered by the user: skip Autoconfig
-                *step_counter = STEP_13_AFTER_AUTOCONFIG - 1; // minus one as step_counter is increased on next loop
-            }
-        }
-        /* A.  Search configurations from the domain used in the email-address, prefer encrypted */
-        5 => {
-            if param_autoconfig.is_none() {
-                let url = format!(
+impl AutoconfigSource {
+    fn all(domain: &str, addr: &str) -> [Self; 5] {
+        [
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!(
                     "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    param_domain, param_addr_urlencoded
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        6 => {
-            progress!(ctx, 300);
-            if param_autoconfig.is_none() {
-                // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-                let url = format!(
+                    domain, addr,
+                ),
+            },
+            // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!(
                     "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-                    param_domain, param_addr_urlencoded
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        /* Outlook section start ------------- */
-        /* Outlook uses always SSL but different domains (this comment describes the next two steps) */
-        7 => {
-            progress!(ctx, 310);
-            if param_autoconfig.is_none() {
-                let url = format!("https://{}/autodiscover/autodiscover.xml", param_domain);
-                *param_autoconfig = outlk_autodiscover(ctx, &url, &param).await.ok();
-            }
-        }
-        8 => {
-            progress!(ctx, 320);
-            if param_autoconfig.is_none() {
-                let url = format!(
+                    domain, addr
+                ),
+            },
+            AutoconfigSource {
+                provider: AutoconfigProvider::Outlook,
+                url: format!("https://{}/autodiscover/autodiscover.xml", domain),
+            },
+            // Outlook uses always SSL but different domains (this comment describes the next two steps)
+            AutoconfigSource {
+                provider: AutoconfigProvider::Outlook,
+                url: format!(
                     "https://{}{}/autodiscover/autodiscover.xml",
-                    "autodiscover.", param_domain
-                );
-                *param_autoconfig = outlk_autodiscover(ctx, &url, &param).await.ok();
-            }
-        }
-        /* ----------- Outlook section end */
-        9 => {
-            progress!(ctx, 330);
-            if param_autoconfig.is_none() {
-                let url = format!(
-                    "http://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    param_domain, param_addr_urlencoded
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        10 => {
-            progress!(ctx, 340);
-            if param_autoconfig.is_none() {
-                // do not transfer the email-address unencrypted
-                let url = format!(
-                    "http://{}/.well-known/autoconfig/mail/config-v1.1.xml",
-                    param_domain
-                );
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        /* B.  If we have no configuration yet, search configuration in Thunderbird's centeral database */
-        11 => {
-            progress!(ctx, 350);
-            if param_autoconfig.is_none() {
-                /* always SSL for Thunderbird's database */
-                let url = format!("https://autoconfig.thunderbird.net/v1.1/{}", param_domain);
-                *param_autoconfig = moz_autoconfigure(ctx, &url, &param).await.ok();
-            }
-        }
-        /* C.  Do we have any autoconfig result?
-           If you change the match-number here, also update STEP_12_COPY_AUTOCONFIG above
-        */
-        STEP_12_USE_AUTOCONFIG => {
-            progress!(ctx, 500);
-            if let Some(ref cfg) = param_autoconfig {
-                info!(ctx, "Got autoconfig: {}", &cfg);
-                if !cfg.mail_user.is_empty() {
-                    param.mail_user = cfg.mail_user.clone();
-                }
-                param.mail_server = cfg.mail_server.clone(); /* all other values are always NULL when entering autoconfig */
-                param.mail_port = cfg.mail_port;
-                param.send_server = cfg.send_server.clone();
-                param.send_port = cfg.send_port;
-                param.send_user = cfg.send_user.clone();
-                param.server_flags = cfg.server_flags;
-                /* although param_autoconfig's data are no longer needed from,
-                it is used to later to prevent trying variations of port/server/logins */
-            }
-            param.server_flags |= *keep_flags;
-        }
-        // Step 3: Fill missing fields with defaults
-        // If you change the match-number here, also update STEP_13_AFTER_AUTOCONFIG above
-        STEP_13_AFTER_AUTOCONFIG => {
-            if param.mail_server.is_empty() {
-                param.mail_server = format!("imap.{}", param_domain,)
-            }
-            if param.mail_port == 0 {
-                param.mail_port = if 0 != param.server_flags & (0x100 | 0x400) {
-                    143
-                } else {
-                    993
-                }
-            }
-            if param.mail_user.is_empty() {
-                param.mail_user = param.addr.clone();
-            }
-            if param.send_server.is_empty() && !param.mail_server.is_empty() {
-                param.send_server = param.mail_server.clone();
-                if param.send_server.starts_with("imap.") {
-                    param.send_server = param.send_server.replacen("imap", "smtp", 1);
-                }
-            }
-            if param.send_port == 0 {
-                param.send_port = if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
-                    587
-                } else if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
-                    25
-                } else {
-                    465
-                }
-            }
-            if param.send_user.is_empty() && !param.mail_user.is_empty() {
-                param.send_user = param.mail_user.clone();
-            }
-            if param.send_pw.is_empty() && !param.mail_pw.is_empty() {
-                param.send_pw = param.mail_pw.clone()
-            }
-            if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
-                param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-                param.server_flags |= DC_LP_AUTH_NORMAL as i32
-            }
-            if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
-                param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-                param.server_flags |= if param.send_port == 143 {
-                    DC_LP_IMAP_SOCKET_STARTTLS as i32
-                } else {
-                    DC_LP_IMAP_SOCKET_SSL as i32
-                }
-            }
-            if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
-                param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-                param.server_flags |= if param.send_port == 587 {
-                    DC_LP_SMTP_SOCKET_STARTTLS as i32
-                } else if param.send_port == 25 {
-                    DC_LP_SMTP_SOCKET_PLAIN as i32
-                } else {
-                    DC_LP_SMTP_SOCKET_SSL as i32
-                }
-            }
-            /* do we have a complete configuration? */
-            if param.mail_server.is_empty()
-                || param.mail_port == 0
-                || param.mail_user.is_empty()
-                || param.mail_pw.is_empty()
-                || param.send_server.is_empty()
-                || param.send_port == 0
-                || param.send_user.is_empty()
-                || param.send_pw.is_empty()
-                || param.server_flags == 0
-            {
-                bail!("Account settings incomplete.");
-            }
-        }
-        14 => {
-            progress!(ctx, 600);
-            /* try to connect to IMAP - if we did not got an autoconfig,
-            do some further tries with different settings and username variations */
-            try_imap_connections(ctx, param, param_autoconfig.is_some(), imap).await?;
-        }
-        15 => {
-            progress!(ctx, 800);
-            try_smtp_connections(ctx, param, param_autoconfig.is_some()).await?;
-        }
-        16 => {
-            progress!(ctx, 900);
+                    "autodiscover.", domain
+                ),
+            },
+            // always SSL for Thunderbird's database
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!("https://autoconfig.thunderbird.net/v1.1/{}", domain),
+            },
+        ]
+    }
 
-            let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
-                || ctx.get_config_bool(Config::MvboxMove).await;
+    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<LoginParam> {
+        let params = match self.provider {
+            AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
+            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url, &param).await?,
+        };
 
-            if let Err(err) = imap.configure_folders(ctx, create_mvbox).await {
-                bail!("configuring folders failed: {:?}", err);
-            }
+        Ok(params)
+    }
+}
 
-            if let Err(err) = imap.select_with_uidvalidity(ctx, "INBOX").await {
-                bail!("could not read INBOX status: {:?}", err);
-            }
-        }
-        17 => {
-            progress!(ctx, 910);
-            // configuration success - write back the configured parameters with the
-            // "configured_" prefix; also write the "configured"-flag */
-            // the trailing underscore is correct
-            param.save_to_database(ctx, "configured_").await?;
-            ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
-        }
-        18 => {
-            progress!(ctx, 920);
-            // we generate the keypair just now - we could also postpone this until the first message is sent, however,
-            // this may result in a unexpected and annoying delay when the user sends his very first message
-            // (~30 seconds on a Moto G4 play) and might looks as if message sending is always that slow.
-            e2ee::ensure_secret_key_exists(ctx).await?;
-            info!(ctx, "key generation completed");
-            progress!(ctx, 940);
-            return Ok(Step::Done);
-        }
-        _ => {
-            bail!("Internal error: step counter out of bound");
+/// Retrieve available autoconfigurations.
+///
+/// A Search configurations from the domain used in the email-address, prefer encrypted
+/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
+async fn get_autoconfig(
+    ctx: &Context,
+    param: &LoginParam,
+    param_domain: &str,
+    param_addr_urlencoded: &str,
+) -> Option<LoginParam> {
+    let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
+
+    let mut progress = 300;
+    for source in &sources {
+        let res = source.fetch(ctx, param).await;
+        progress!(ctx, progress);
+        progress += 10;
+        if let Ok(res) = res {
+            return Some(res);
         }
     }
 
-    Ok(Step::Continue)
+    None
 }
 
-#[derive(Debug)]
-enum Step {
-    Done,
-    Continue,
-}
-
-#[allow(clippy::unnecessary_unwrap)]
 fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
     info!(
         context,
@@ -479,37 +403,35 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
                 // however, rewriting the code to "if let" would make things less obvious,
                 // esp. if we allow more combinations of servers (pop, jmap).
                 // therefore, #[allow(clippy::unnecessary_unwrap)] is added above.
-                if imap.is_some() && smtp.is_some() {
-                    let imap = imap.unwrap();
-                    let smtp = smtp.unwrap();
+                if let Some(imap) = imap {
+                    if let Some(smtp) = smtp {
+                        let mut p = LoginParam::new();
+                        p.addr = param.addr.clone();
 
-                    let mut p = LoginParam::new();
-                    p.addr = param.addr.clone();
+                        p.mail_server = imap.hostname.to_string();
+                        p.mail_user = imap.apply_username_pattern(param.addr.clone());
+                        p.mail_port = imap.port as i32;
+                        p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.server_flags |= match imap.socket {
+                            provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
+                            provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
+                        };
 
-                    p.mail_server = imap.hostname.to_string();
-                    p.mail_user = imap.apply_username_pattern(param.addr.clone());
-                    p.mail_port = imap.port as i32;
-                    p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                    p.server_flags |= match imap.socket {
-                        provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
-                        provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
-                    };
+                        p.send_server = smtp.hostname.to_string();
+                        p.send_user = smtp.apply_username_pattern(param.addr.clone());
+                        p.send_port = smtp.port as i32;
+                        p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.server_flags |= match smtp.socket {
+                            provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
+                            provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
+                        };
 
-                    p.send_server = smtp.hostname.to_string();
-                    p.send_user = smtp.apply_username_pattern(param.addr.clone());
-                    p.send_port = smtp.port as i32;
-                    p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                    p.server_flags |= match smtp.socket {
-                        provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
-                        provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
-                    };
-
-                    info!(context, "offline autoconfig found: {}", p);
-                    return Some(p);
-                } else {
-                    info!(context, "offline autoconfig found, but no servers defined");
-                    return None;
+                        info!(context, "offline autoconfig found: {}", p);
+                        return Some(p);
+                    }
                 }
+                info!(context, "offline autoconfig found, but no servers defined");
+                return None;
             }
             provider::Status::BROKEN => {
                 info!(context, "offline autoconfig found, provider is broken");
@@ -523,19 +445,32 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
 
 async fn try_imap_connections(
     context: &Context,
-    mut param: &mut LoginParam,
+    param: &mut LoginParam,
     was_autoconfig: bool,
     imap: &mut Imap,
-) -> Result<bool> {
-    // progress 650 and 660
-    if let Ok(val) = try_imap_connection(context, &mut param, was_autoconfig, 0, imap).await {
-        return Ok(val);
-    }
-    progress!(context, 670);
-    param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-    param.server_flags |= DC_LP_IMAP_SOCKET_SSL;
-    param.mail_port = 993;
+) -> Result<()> {
+    // manually_set_param is used to check whether a particular setting was set manually by the user.
+    // If yes, we do not want to change it to avoid confusing error messages
+    // (you set port 443, but the app tells you it couldn't connect on port 993).
+    let manually_set_param = LoginParam::from_database(context, "").await;
 
+    // progress 650 and 660
+    if try_imap_connection(context, param, &manually_set_param, was_autoconfig, 0, imap)
+        .await
+        .is_ok()
+    {
+        return Ok(()); // we directly return here if it was autoconfig or the connection succeeded
+    }
+
+    progress!(context, 670);
+    // try_imap_connection() changed the flags and port. Change them back:
+    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
+        param.server_flags |= DC_LP_IMAP_SOCKET_SSL;
+    }
+    if manually_set_param.mail_port == 0 {
+        param.mail_port = 993;
+    }
     if let Some(at) = param.mail_user.find('@') {
         param.mail_user = param.mail_user.split_at(at).0.to_string();
     }
@@ -543,35 +478,43 @@ async fn try_imap_connections(
         param.send_user = param.send_user.split_at(at).0.to_string();
     }
     // progress 680 and 690
-    try_imap_connection(context, &mut param, was_autoconfig, 1, imap).await
+    try_imap_connection(context, param, &manually_set_param, was_autoconfig, 1, imap).await
 }
 
 async fn try_imap_connection(
     context: &Context,
     param: &mut LoginParam,
+    manually_set_param: &LoginParam,
     was_autoconfig: bool,
     variation: usize,
     imap: &mut Imap,
-) -> Result<bool> {
-    if try_imap_one_param(context, &param, imap).await.is_ok() {
-        return Ok(true);
+) -> Result<()> {
+    if try_imap_one_param(context, param, imap).await.is_ok() {
+        return Ok(());
     }
     if was_autoconfig {
-        return Ok(false);
+        return Ok(());
     }
+
     progress!(context, 650 + variation * 30);
-    param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-    param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS;
-    if try_imap_one_param(context, &param, imap).await.is_ok() {
-        return Ok(true);
+
+    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
+        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS;
+
+        if try_imap_one_param(context, &param, imap).await.is_ok() {
+            return Ok(());
+        }
     }
 
     progress!(context, 660 + variation * 30);
-    param.mail_port = 143;
 
-    try_imap_one_param(context, &param, imap).await?;
-
-    Ok(true)
+    if manually_set_param.mail_port == 0 {
+        param.mail_port = 143;
+        try_imap_one_param(context, param, imap).await
+    } else {
+        Err(format_err!("no more possible configs"))
+    }
 }
 
 async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Imap) -> Result<()> {
@@ -590,41 +533,51 @@ async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Im
         return Ok(());
     }
 
-    if context.shall_stop_ongoing().await {
-        bail!("Interrupted");
-    }
-
     bail!("Could not connect: {}", inf);
 }
 
 async fn try_smtp_connections(
     context: &Context,
-    mut param: &mut LoginParam,
+    param: &mut LoginParam,
     was_autoconfig: bool,
 ) -> Result<()> {
+    // manually_set_param is used to check whether a particular setting was set manually by the user.
+    // If yes, we do not want to change it to avoid confusing error messages
+    // (you set port 443, but the app tells you it couldn't connect on port 993).
+    let manually_set_param = LoginParam::from_database(context, "").await;
+
     let mut smtp = Smtp::new();
-    /* try to connect to SMTP - if we did not got an autoconfig, the first try was SSL-465 and we do a second try with STARTTLS-587 */
-    if try_smtp_one_param(context, &param, &mut smtp).await.is_ok() {
+    // try to connect to SMTP - if we did not got an autoconfig, the first try was SSL-465 and we do
+    // a second try with STARTTLS-587
+    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
         return Ok(());
     }
     if was_autoconfig {
         return Ok(());
     }
     progress!(context, 850);
-    param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-    param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    param.send_port = 587;
 
-    if try_smtp_one_param(context, &param, &mut smtp).await.is_ok() {
+    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
+    }
+    if manually_set_param.send_port == 0 {
+        param.send_port = 587;
+    }
+
+    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
         return Ok(());
     }
     progress!(context, 860);
-    param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-    param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    param.send_port = 25;
-    try_smtp_one_param(context, &param, &mut smtp).await?;
 
-    Ok(())
+    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
+    }
+    if manually_set_param.send_port == 0 {
+        param.send_port = 25;
+    }
+    try_smtp_one_param(context, param, &mut smtp).await
 }
 
 async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Smtp) -> Result<()> {
@@ -674,7 +627,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_no_panic_on_bad_credentials() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::Addr, Some("probably@unexistant.addr"))
             .await
@@ -688,7 +641,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_offline_autoconfig() {
-        let context = dummy_context().await.ctx;
+        let context = TestContext::new().await.ctx;
 
         let mut params = LoginParam::new();
         params.addr = "someone123@example.org".to_string();

src/constants.rs

@@ -227,6 +227,10 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
+// max. width/height of images
+pub const BALANCED_IMAGE_SIZE: u32 = 1280;
+pub const WORSE_IMAGE_SIZE: u32 = 640;
+
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
@@ -319,54 +323,6 @@ const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
 const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
 const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
 const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
-
-// TODO: Strings need some doumentation about used placeholders.
-// These constants are used to set stock translation strings
-
-const DC_STR_NOMESSAGES: usize = 1;
-const DC_STR_SELF: usize = 2;
-const DC_STR_DRAFT: usize = 3;
-const DC_STR_VOICEMESSAGE: usize = 7;
-const DC_STR_DEADDROP: usize = 8;
-const DC_STR_IMAGE: usize = 9;
-const DC_STR_VIDEO: usize = 10;
-const DC_STR_AUDIO: usize = 11;
-const DC_STR_FILE: usize = 12;
-const DC_STR_STATUSLINE: usize = 13;
-const DC_STR_NEWGROUPDRAFT: usize = 14;
-const DC_STR_MSGGRPNAME: usize = 15;
-const DC_STR_MSGGRPIMGCHANGED: usize = 16;
-const DC_STR_MSGADDMEMBER: usize = 17;
-const DC_STR_MSGDELMEMBER: usize = 18;
-const DC_STR_MSGGROUPLEFT: usize = 19;
-const DC_STR_GIF: usize = 23;
-const DC_STR_ENCRYPTEDMSG: usize = 24;
-const DC_STR_E2E_AVAILABLE: usize = 25;
-const DC_STR_ENCR_TRANSP: usize = 27;
-const DC_STR_ENCR_NONE: usize = 28;
-const DC_STR_CANTDECRYPT_MSG_BODY: usize = 29;
-const DC_STR_FINGERPRINTS: usize = 30;
-const DC_STR_READRCPT: usize = 31;
-const DC_STR_READRCPT_MAILBODY: usize = 32;
-const DC_STR_MSGGRPIMGDELETED: usize = 33;
-const DC_STR_E2E_PREFERRED: usize = 34;
-const DC_STR_CONTACT_VERIFIED: usize = 35;
-const DC_STR_CONTACT_NOT_VERIFIED: usize = 36;
-const DC_STR_CONTACT_SETUP_CHANGED: usize = 37;
-const DC_STR_ARCHIVEDCHATS: usize = 40;
-const DC_STR_STARREDMSGS: usize = 41;
-const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
-const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_CANNOT_LOGIN: usize = 60;
-const DC_STR_SERVER_RESPONSE: usize = 61;
-const DC_STR_MSGACTIONBYUSER: usize = 62;
-const DC_STR_MSGACTIONBYME: usize = 63;
-const DC_STR_MSGLOCATIONENABLED: usize = 64;
-const DC_STR_MSGLOCATIONDISABLED: usize = 65;
-const DC_STR_LOCATION: usize = 66;
-const DC_STR_STICKER: usize = 67;
-const DC_STR_COUNT: usize = 67;
 
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 

src/contact.rs

@@ -1,8 +1,12 @@
 //! Contacts module
 
+#![forbid(clippy::indexing_slicing)]
+
 use async_std::path::PathBuf;
 use deltachat_derive::*;
 use itertools::Itertools;
+use lazy_static::lazy_static;
+use regex::Regex;
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
@@ -12,7 +16,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
 use crate::mimeparser::AvatarAction;
@@ -238,6 +242,8 @@ impl Contact {
             "Cannot create contact with empty address"
         );
 
+        let (name, addr) = sanitize_name_and_addr(name, addr);
+
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
@@ -512,8 +518,9 @@ impl Contact {
         let mut modify_cnt = 0;
 
         for (name, addr) in split_address_book(addr_book.as_ref()).into_iter() {
+            let (name, addr) = sanitize_name_and_addr(name, addr);
             let name = normalize_name(name);
-            match Contact::add_or_lookup(context, name, addr, Origin::AddressBook).await {
+            match Contact::add_or_lookup(context, name, &addr, Origin::AddressBook).await {
                 Err(err) => {
                     warn!(
                         context,
@@ -691,18 +698,20 @@ impl Contact {
                     })
                     .await;
                 ret += &p;
-                let self_key = Key::from(SignedPublicKey::load_self(context).await?);
                 let p = context.stock_str(StockMessage::FingerPrints).await;
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key.formatted_fingerprint();
+                let fingerprint_self = SignedPublicKey::load_self(context)
+                    .await?
+                    .fingerprint()
+                    .to_string();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
-                    .map(|k| k.formatted_fingerprint())
+                    .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 let fingerprint_other_unverified = peerstate
                     .peek_key(PeerstateVerifiedStatus::Unverified)
-                    .map(|k| k.formatted_fingerprint())
+                    .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 if loginparam.addr < peerstate.addr {
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
@@ -1022,10 +1031,32 @@ pub fn addr_normalize(addr: &str) -> &str {
     let norm = addr.trim();
 
     if norm.starts_with("mailto:") {
-        return &norm[7..];
+        norm.get(7..).unwrap_or(norm)
+    } else {
+        norm
     }
+}
 
-    norm
+fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
+    lazy_static! {
+        static ref ADDR_WITH_NAME_REGEX: Regex = Regex::new("(.*)<(.*)>").unwrap();
+    }
+    if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
+        (
+            if name.as_ref().is_empty() {
+                captures
+                    .get(1)
+                    .map_or("".to_string(), |m| normalize_name(m.as_str()))
+            } else {
+                name.as_ref().to_string()
+            },
+            captures
+                .get(2)
+                .map_or("".to_string(), |m| m.as_str().to_string()),
+        )
+    } else {
+        (name.as_ref().to_string(), addr.as_ref().to_string())
+    }
 }
 
 async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
@@ -1088,38 +1119,21 @@ pub(crate) async fn set_profile_image(
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
-/// - Convert names as "Petersen, Björn" to "Björn Petersen"
 /// - Trims the resulting string
 ///
 /// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
 pub fn normalize_name(full_name: impl AsRef<str>) -> String {
-    let mut full_name = full_name.as_ref().trim();
+    let full_name = full_name.as_ref().trim();
     if full_name.is_empty() {
         return full_name.into();
     }
 
-    let len = full_name.len();
-    if len > 1 {
-        let firstchar = full_name.as_bytes()[0];
-        let lastchar = full_name.as_bytes()[len - 1];
-        if firstchar == b'\'' && lastchar == b'\''
-            || firstchar == b'\"' && lastchar == b'\"'
-            || firstchar == b'<' && lastchar == b'>'
-        {
-            full_name = &full_name[1..len - 1];
-        }
+    match full_name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => full_name
+            .get(1..full_name.len() - 1)
+            .map_or("".to_string(), |s| s.trim().into()),
+        _ => full_name.to_string(),
     }
-
-    if let Some(p1) = full_name.find(',') {
-        let (last_name, first_name) = full_name.split_at(p1);
-
-        let last_name = last_name.trim();
-        let first_name = (&first_name[1..]).trim();
-
-        return format!("{} {}", first_name, last_name);
-    }
-
-    full_name.trim().into()
 }
 
 fn cat_fingerprint(
@@ -1202,11 +1216,14 @@ mod tests {
         assert_eq!(may_be_valid_addr("u@d.tt"), true);
         assert_eq!(may_be_valid_addr("u@.tt"), false);
         assert_eq!(may_be_valid_addr("@d.tt"), false);
+        assert_eq!(may_be_valid_addr("<da@d.tt"), false);
+        assert_eq!(may_be_valid_addr("sk <@d.tt>"), false);
+        assert_eq!(may_be_valid_addr("as@sd.de>"), false);
+        assert_eq!(may_be_valid_addr("ask dkl@dd.tt"), false);
     }
 
     #[test]
     fn test_normalize_name() {
-        assert_eq!(&normalize_name("Doe, John"), "John Doe");
         assert_eq!(&normalize_name(" hello world   "), "hello world");
         assert_eq!(&normalize_name("<"), "<");
         assert_eq!(&normalize_name(">"), ">");
@@ -1241,7 +1258,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_contacts() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let contacts = Contact::get_all(&context.ctx, 0, Some("some2"))
             .await
             .unwrap();
@@ -1265,10 +1282,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_self_addr() -> Result<()> {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         assert!(t.ctx.is_self_addr("me@me.org").await.is_err());
 
-        let addr = configure_alice_keypair(&t.ctx).await;
+        let addr = t.configure_alice().await;
         assert_eq!(t.ctx.is_self_addr("me@me.org").await?, false);
         assert_eq!(t.ctx.is_self_addr(&addr).await?, true);
 
@@ -1278,15 +1295,16 @@ mod tests {
     #[async_std::test]
     async fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let book = concat!(
             "  Name one  \n one@eins.org \n",
             "Name two\ntwo@deux.net\n",
             "Invalid\n+1234567890\n", // invalid, should be ignored
             "\nthree@drei.sam\n",
-            "Name two\ntwo@deux.net\n" // should not be added again
+            "Name two\ntwo@deux.net\n", // should not be added again
+            "\nWonderland, Alice <alice@w.de>\n",
         );
-        assert_eq!(Contact::add_address_book(&t.ctx, book).await.unwrap(), 3);
+        assert_eq!(Contact::add_address_book(&t.ctx, book).await.unwrap(), 4);
 
         // check first added contact, this does not modify because of lower origin
         let (contact_id, sth_modified) =
@@ -1362,6 +1380,19 @@ mod tests {
         assert_eq!(contact.get_name_n_addr(), "schnucki (three@drei.sam)");
         assert!(!contact.is_blocked());
 
+        // Fourth contact:
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "", "alice@w.de", Origin::IncomingUnknownTo)
+                .await
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::None);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "Wonderland, Alice");
+        assert_eq!(contact.get_display_name(), "Wonderland, Alice");
+        assert_eq!(contact.get_addr(), "alice@w.de");
+        assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
+
         // check SELF
         let contact = Contact::load_from_db(&t.ctx, DC_CONTACT_ID_SELF)
             .await
@@ -1377,7 +1408,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
@@ -1440,7 +1471,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_create_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t.ctx, "", "claire@example.org")
@@ -1487,7 +1518,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_edit_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "dave@example.org"
         let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org")
@@ -1528,4 +1559,50 @@ mod tests {
         assert!(addr_cmp(" aa@aa.ORG ", "AA@AA.ORG"));
         assert!(addr_cmp(" mailto:AA@AA.ORG", "Aa@Aa.orG"));
     }
+
+    #[async_std::test]
+    async fn test_name_in_address() {
+        let t = TestContext::new().await;
+
+        let contact_id = Contact::create(&t.ctx, "", "<dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        let contact_id = Contact::create(&t.ctx, "", "Mueller, Dave <dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "Mueller, Dave");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        let contact_id = Contact::create(&t.ctx, "name1", "name2 <dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "name1");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        assert!(Contact::create(&t.ctx, "", "<dskjfdslk@sadklj.dk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "<dskjf>dslk@sadklj.dk>")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjfdslksadklj.dk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjfdslk@sadklj.dk>")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjf@dslk@sadkljdk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjf dslk@d.e").await.is_err());
+        assert!(Contact::create(&t.ctx, "", "<dskjf dslk@sadklj.dk")
+            .await
+            .is_err());
+    }
 }

src/context.rs

@@ -6,6 +6,7 @@ use std::ops::Deref;
 
 use async_std::path::{Path, PathBuf};
 use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use async_std::task;
 
 use crate::chat::*;
 use crate::config::Config;
@@ -15,7 +16,7 @@ use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::{Event, EventEmitter, Events};
 use crate::job::{self, Action};
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
 use crate::message::{self, Message, MessengerMessage, MsgId};
@@ -56,6 +57,7 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
     creation_time: SystemTime,
 }
@@ -121,6 +123,7 @@ impl Context {
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
         };
 
@@ -138,10 +141,15 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        assert!(!self.is_io_running().await, "context is already running");
+        if self.is_io_running().await {
+            info!(self, "IO is already running");
+            return;
+        }
 
-        let l = &mut *self.inner.scheduler.write().await;
-        l.start(self.clone()).await;
+        {
+            let l = &mut *self.inner.scheduler.write().await;
+            l.start(self.clone()).await;
+        }
     }
 
     /// Returns if the IO scheduler is running.
@@ -152,6 +160,11 @@ impl Context {
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
+        if !self.is_io_running().await {
+            info!(self, "IO is not running");
+            return;
+        }
+
         self.inner.stop_io().await;
     }
 
@@ -275,7 +288,7 @@ impl Context {
             .query_get_value(self, "SELECT COUNT(*) FROM acpeerstates;", paramsv![])
             .await;
         let fingerprint_str = match SignedPublicKey::load_self(self).await {
-            Ok(key) => Key::from(key).fingerprint(),
+            Ok(key) => key.fingerprint().hex(),
             Err(err) => format!("<key failure: {}>", err),
         };
 
@@ -290,13 +303,11 @@ impl Context {
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
+            .get_config(Config::ConfiguredSentboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
+            .get_config(Config::ConfiguredMvboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
 
@@ -432,33 +443,19 @@ impl Context {
             .unwrap_or_default()
     }
 
-    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        folder_name.as_ref() == "INBOX"
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
-            .await;
-        if let Some(name) = sentbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
-            .await;
-
-        if let Some(name) = mvbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
@@ -546,7 +543,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_fresh_msgs() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let fresh = t.ctx.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
@@ -601,13 +598,13 @@ mod tests {
 
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         std::mem::drop(t.ctx);
     }
 
     #[async_std::test]
     async fn test_get_info() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let info = t.ctx.get_info().await;
         assert!(info.get("database_dir").is_some());

src/dc_receive_imf.rs

@@ -10,7 +10,8 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, Result};
+use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer};
+use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
@@ -45,13 +46,14 @@ pub async fn dc_receive_imf(
 ) -> Result<()> {
     info!(
         context,
-        "Receiving message {}/{}...",
+        "Receiving message {}/{}, seen={}...",
         if !server_folder.as_ref().is_empty() {
             server_folder.as_ref()
         } else {
             "?"
         },
         server_uid,
+        seen
     );
 
     if std::env::var(crate::DCC_MIME_DEBUG).unwrap_or_default() == "2" {
@@ -227,6 +229,19 @@ pub async fn dc_receive_imf(
             context
                 .do_heuristics_moves(server_folder.as_ref(), insert_msg_id)
                 .await;
+            if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
+                // This is a Delta Chat MDN. Mark as read.
+                job::add(
+                    context,
+                    job::Job::new(
+                        Action::MarkseenMsgOnImap,
+                        insert_msg_id.to_u32(),
+                        Params::new(),
+                        0,
+                    ),
+                )
+                .await;
+            }
         }
     }
 
@@ -238,7 +253,7 @@ pub async fn dc_receive_imf(
     cleanup(context, &create_event_to_send, created_db_entries);
 
     mime_parser
-        .handle_reports(context, from_id, sent_timestamp)
+        .handle_reports(context, from_id, sent_timestamp, &mime_parser.parts)
         .await;
 
     Ok(())
@@ -313,8 +328,6 @@ async fn add_parts(
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
-    let mut sort_timestamp = 0;
-    let mut rcvd_timestamp = 0;
     let mut mime_in_reply_to = String::new();
     let mut mime_references = String::new();
     let mut incoming_origin = incoming_origin;
@@ -334,7 +347,7 @@ async fn add_parts(
         return Ok(());
     }
 
-    let mut msgrmsg = if mime_parser.has_chat_version() {
+    let mut is_dc_message = if mime_parser.has_chat_version() {
         MessengerMessage::Yes
     } else if is_reply_to_messenger_message(context, mime_parser).await {
         MessengerMessage::Reply
@@ -346,7 +359,7 @@ async fn add_parts(
     let show_emails =
         ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await).unwrap_or_default();
     if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage
-        && msgrmsg == MessengerMessage::No
+        && is_dc_message == MessengerMessage::No
     {
         // this message is a classic email not a chat-message nor a reply to one
         match show_emails {
@@ -375,7 +388,7 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get(HeaderDef::SecureJoin).is_some() {
-            msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting
+            is_dc_message = MessengerMessage::Yes; // avoid discarding by show_emails setting
             *chat_id = ChatId::new(0);
             allow_creation = true;
             match handle_securejoin_handshake(context, mime_parser, from_id).await {
@@ -407,6 +420,14 @@ async fn add_parts(
                 .await
                 .unwrap_or_default();
 
+        if chat_id.is_unset() && mime_parser.failure_report.is_some() {
+            *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+            info!(
+                context,
+                "Message belongs to an NDN and is not shown in a chat.",
+            );
+        }
+
         // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
         // it might also be blocked and displayed in the deaddrop as a result
         if chat_id.is_unset() {
@@ -500,7 +521,7 @@ async fn add_parts(
         if Blocked::Not != chat_id_blocked
             && state == MessageState::InFresh
             && !incoming_origin.is_known()
-            && msgrmsg == MessengerMessage::No
+            && is_dc_message == MessengerMessage::No
             && show_emails != ShowEmails::All
         {
             state = MessageState::InNoticed;
@@ -515,7 +536,7 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get(HeaderDef::SecureJoin).is_some() {
-            msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting
+            is_dc_message = MessengerMessage::Yes; // avoid discarding by show_emails setting
             *chat_id = ChatId::new(0);
             allow_creation = true;
             match observe_securejoin_on_other_device(context, mime_parser, to_id).await {
@@ -554,7 +575,7 @@ async fn add_parts(
                 }
             }
             if chat_id.is_unset() && allow_creation {
-                let create_blocked = if MessengerMessage::No != msgrmsg
+                let create_blocked = if MessengerMessage::No != is_dc_message
                     && !Contact::is_blocked_load(context, to_id).await
                 {
                     Blocked::Not
@@ -599,19 +620,86 @@ async fn add_parts(
             *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
         }
     }
+
+    // Extract ephemeral timer from the message.
+    let mut timer = if let Some(value) = mime_parser.get(HeaderDef::EphemeralTimer) {
+        match value.parse::<EphemeralTimer>() {
+            Ok(timer) => timer,
+            Err(err) => {
+                warn!(
+                    context,
+                    "can't parse ephemeral timer \"{}\": {}", value, err
+                );
+                EphemeralTimer::Disabled
+            }
+        }
+    } else {
+        EphemeralTimer::Disabled
+    };
+
+    let location_kml_is = mime_parser.location_kml.is_some();
+    let is_mdn = !mime_parser.mdn_reports.is_empty();
+
+    // Apply ephemeral timer changes to the chat.
+    //
+    // Only non-hidden timers are applied now. Timers from hidden
+    // messages such as read receipts can be useful to detect
+    // ephemeral timer support, but timer changes without visible
+    // received messages may be confusing to the user.
+    if !*hidden
+        && !location_kml_is
+        && !is_mdn
+        && (*chat_id).get_ephemeral_timer(context).await? != timer
+    {
+        match (*chat_id).inner_set_ephemeral_timer(context, timer).await {
+            Ok(()) => {
+                if mime_parser.is_system_message == SystemMessage::EphemeralTimerChanged {
+                    set_better_msg(
+                        mime_parser,
+                        stock_ephemeral_timer_changed(context, timer, from_id).await,
+                    );
+
+                    // Do not delete the system message itself.
+                    //
+                    // This prevents confusion when timer is changed
+                    // to 1 week, and then changed to 1 hour: after 1
+                    // hour, only the message about the change to 1
+                    // week is left.
+                    timer = EphemeralTimer::Disabled;
+                } else {
+                    chat::add_info_msg(
+                        context,
+                        *chat_id,
+                        stock_ephemeral_timer_changed(context, timer, from_id).await,
+                    )
+                    .await;
+                }
+
+                context.emit_event(Event::ChatEphemeralTimerModified {
+                    chat_id: *chat_id,
+                    timer: timer.to_u32(),
+                });
+            }
+            Err(err) => {
+                warn!(
+                    context,
+                    "failed to modify timer for chat {}: {}", chat_id, err
+                );
+            }
+        }
+    }
+
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
-    calc_timestamps(
+    let rcvd_timestamp = time();
+    let sort_timestamp = calc_sort_timestamp(
         context,
-        *chat_id,
-        from_id,
         *sent_timestamp,
-        !seen,
-        &mut sort_timestamp,
-        sent_timestamp,
-        &mut rcvd_timestamp,
+        *chat_id,
+        state == MessageState::InFresh,
     )
     .await;
+    *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
     chat_id.unarchive(context).await?;
@@ -637,7 +725,6 @@ async fn add_parts(
 
     let mut parts = std::mem::replace(&mut mime_parser.parts, Vec::new());
     let server_folder = server_folder.as_ref().to_string();
-    let location_kml_is = mime_parser.location_kml.is_some();
     let is_system_message = mime_parser.is_system_message;
     let mime_headers = if save_mime_headers {
         Some(String::from_utf8_lossy(imf_raw).to_string())
@@ -647,7 +734,6 @@ async fn add_parts(
     let sent_timestamp = *sent_timestamp;
     let is_hidden = *hidden;
     let chat_id = *chat_id;
-    let is_mdn = !mime_parser.reports.is_empty();
 
     // TODO: can this clone be avoided?
     let rfc724_mid = rfc724_mid.to_string();
@@ -664,8 +750,8 @@ async fn add_parts(
                     "INSERT INTO msgs \
          (rfc724_mid, server_folder, server_uid, chat_id, from_id, to_id, timestamp, \
          timestamp_sent, timestamp_rcvd, type, state, msgrmsg,  txt, txt_raw, param, \
-         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references) \
-         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?);",
+         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references, error, ephemeral_timer) \
+         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?, ?,?);",
                 )?;
 
                 let is_location_kml = location_kml_is
@@ -699,7 +785,7 @@ async fn add_parts(
                     rcvd_timestamp,
                     part.typ,
                     state,
-                    msgrmsg,
+                    is_dc_message,
                     part.msg,
                     // txt_raw might contain invalid utf8
                     txt_raw,
@@ -709,6 +795,8 @@ async fn add_parts(
                     mime_headers,
                     mime_in_reply_to,
                     mime_references,
+                    part.error,
+                    timer
                 ])?;
 
                 drop(stmt);
@@ -749,6 +837,31 @@ async fn add_parts(
         }
     }
 
+    async fn update_last_subject(
+        context: &Context,
+        chat_id: ChatId,
+        mime_parser: &MimeMessage,
+    ) -> Result<()> {
+        let mut chat = Chat::load_from_db(context, chat_id).await?;
+        chat.param.set(
+            Param::LastSubject,
+            mime_parser
+                .get_subject()
+                .ok_or_else(|| format_err!("No subject in email"))?,
+        );
+        chat.update_param(context).await?;
+        Ok(())
+    }
+    update_last_subject(context, chat_id, mime_parser)
+        .await
+        .unwrap_or_else(|e| {
+            warn!(
+                context,
+                "Could not update LastSubject of chat: {}",
+                e.to_string()
+            )
+        });
+
     Ok(())
 }
 
@@ -812,41 +925,38 @@ async fn save_locations(
     }
 }
 
-#[allow(clippy::too_many_arguments)]
-async fn calc_timestamps(
+async fn calc_sort_timestamp(
     context: &Context,
-    chat_id: ChatId,
-    from_id: u32,
     message_timestamp: i64,
+    chat_id: ChatId,
     is_fresh_msg: bool,
-    sort_timestamp: &mut i64,
-    sent_timestamp: &mut i64,
-    rcvd_timestamp: &mut i64,
-) {
-    *rcvd_timestamp = time();
-    *sent_timestamp = message_timestamp;
-    if *sent_timestamp > *rcvd_timestamp {
-        *sent_timestamp = *rcvd_timestamp
-    }
-    *sort_timestamp = message_timestamp;
+) -> i64 {
+    let mut sort_timestamp = message_timestamp;
+
+    // get newest non fresh message for this chat
+    // update sort_timestamp if less than that
     if is_fresh_msg {
         let last_msg_time: Option<i64> = context
             .sql
             .query_get_value(
                 context,
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? and from_id!=? AND timestamp>=?",
-                paramsv![chat_id, from_id as i32, *sort_timestamp],
+                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
+                paramsv![chat_id, MessageState::InFresh],
             )
             .await;
+
         if let Some(last_msg_time) = last_msg_time {
-            if last_msg_time > 0 && *sort_timestamp <= last_msg_time {
-                *sort_timestamp = last_msg_time + 1;
+            if last_msg_time > sort_timestamp {
+                sort_timestamp = last_msg_time;
             }
         }
     }
-    if *sort_timestamp >= dc_smeared_time(context).await {
-        *sort_timestamp = dc_create_smeared_timestamp(context).await;
+
+    if sort_timestamp >= dc_smeared_time(context).await {
+        sort_timestamp = dc_create_smeared_timestamp(context).await;
     }
+
+    sort_timestamp
 }
 
 /// This function tries extracts the group-id from the message and returns the
@@ -889,32 +999,29 @@ async fn create_or_lookup_group(
     }
 
     if grpid.is_empty() {
-        if let Some(value) = mime_parser.get(HeaderDef::MessageId) {
-            if let Some(extracted_grpid) = dc_extract_grpid_from_rfc724_mid(&value) {
-                grpid = extracted_grpid.to_string();
-            }
-        }
-        if grpid.is_empty() {
-            if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
-                grpid = extracted_grpid.to_string();
-            } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References)
-            {
-                grpid = extracted_grpid.to_string();
-            } else {
-                return create_or_lookup_adhoc_group(
-                    context,
-                    mime_parser,
-                    allow_creation,
-                    create_blocked,
-                    from_id,
-                    to_ids,
-                )
-                .await
-                .map_err(|err| {
-                    info!(context, "could not create adhoc-group: {:?}", err);
-                    err
-                });
-            }
+        if let Some(extracted_grpid) = mime_parser
+            .get(HeaderDef::MessageId)
+            .and_then(|value| dc_extract_grpid_from_rfc724_mid(&value))
+        {
+            grpid = extracted_grpid.to_string();
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
+            grpid = extracted_grpid.to_string();
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) {
+            grpid = extracted_grpid.to_string();
+        } else {
+            return create_or_lookup_adhoc_group(
+                context,
+                mime_parser,
+                allow_creation,
+                create_blocked,
+                from_id,
+                to_ids,
+            )
+            .await
+            .map_err(|err| {
+                info!(context, "could not create adhoc-group: {:?}", err);
+                err
+            });
         }
     }
     // now we have a grpid that is non-empty
@@ -1001,7 +1108,7 @@ async fn create_or_lookup_group(
     let (mut chat_id, chat_id_verified, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
         .await
         .unwrap_or((ChatId::new(0), false, Blocked::Not));
-    if !chat_id.is_error() {
+    if !chat_id.is_unset() {
         if chat_id_verified {
             if let Err(err) =
                 check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
@@ -1032,7 +1139,7 @@ async fn create_or_lookup_group(
         .await
         .unwrap_or_default();
 
-    if chat_id.is_error()
+    if chat_id.is_unset()
             && !mime_parser.is_mailinglist_message()
             && !grpid.is_empty()
             && grpname.is_some()
@@ -1205,10 +1312,6 @@ async fn create_or_lookup_adhoc_group(
     from_id: u32,
     to_ids: &ContactIds,
 ) -> Result<(ChatId, Blocked)> {
-    // if we're here, no grpid was found, check if there is an existing
-    // ad-hoc group matching the to-list or if we should and can create one
-    // (we do not want to heuristically look at the likely mangled Subject)
-
     if mime_parser.is_mailinglist_message() {
         // XXX we could parse List-* headers and actually create and
         // manage a mailing list group, eventually
@@ -1219,6 +1322,10 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
+    // if we're here, no grpid was found, check if there is an existing
+    // ad-hoc group matching the to-list or if we should and can create one
+    // (we do not want to heuristically look at the likely mangled Subject)
+
     let mut member_ids: Vec<u32> = to_ids.iter().copied().collect();
     if !member_ids.contains(&from_id) {
         member_ids.push(from_id);
@@ -1271,6 +1378,24 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
+    if mime_parser.decrypting_failed {
+        // Do not create a new ad-hoc group if the message cannot be
+        // decrypted.
+        //
+        // The subject may be encrypted and contain a placeholder such
+        // as "...". Besides that, it is possible that the message was
+        // sent to a valid, yet unknown group, which was rejected
+        // because Chat-Group-Name, which is in the encrypted part,
+        // was not found. Generating a new ID in this case would
+        // result in creation of a twin group with a different group
+        // ID.
+        warn!(
+            context,
+            "not creating ad-hoc group for message that cannot be decrypted"
+        );
+        return Ok((ChatId::new(0), Blocked::Not));
+    }
+
     // we do not check if the message is a reply to another group, this may result in
     // chats with unclear member list. instead we create a new group in the following lines ...
 
@@ -1737,10 +1862,10 @@ fn dc_create_incoming_rfc724_mid(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat::ChatVisibility;
+    use crate::chat::{ChatItem, ChatVisibility};
     use crate::chatlist::Chatlist;
     use crate::message::Message;
-    use crate::test_utils::{dummy_context, TestContext};
+    use crate::test_utils::*;
 
     #[test]
     fn test_hex_hash() {
@@ -1752,7 +1877,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_grpid_simple() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello\n\
                     Subject: outer-subject\n\
                     In-Reply-To: <lqkjwelq123@123123>\n\
@@ -1769,7 +1894,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_grpid_from_multiple() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello\n\
                     Subject: outer-subject\n\
                     In-Reply-To: <Gr.HcxyMARjyJy.9-qweqwe@asd.net>\n\
@@ -1806,7 +1931,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_known_rfc724_mid() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("first message".to_string());
         let msg_id = chat::add_device_msg(&t.ctx, None, Some(&mut msg))
@@ -1822,7 +1947,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_msgrmsg_rfc724_mid() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("first message".to_string());
         let msg_id = chat::add_device_msg(&t.ctx, None, Some(&mut msg))
@@ -1836,51 +1961,34 @@ mod tests {
         assert!(!is_msgrmsg_rfc724_mid(&t.ctx, "nonexistant@message.id").await);
     }
 
-    async fn configured_offline_context() -> TestContext {
-        let t = dummy_context().await;
-        t.ctx
-            .set_config(Config::Addr, Some("alice@example.org"))
-            .await
-            .unwrap();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.org"))
-            .await
-            .unwrap();
-        t.ctx
-            .set_config(Config::Configured, Some("1"))
-            .await
-            .unwrap();
-        t
-    }
-
-    static MSGRMSG: &[u8] = b"From: Bob <bob@example.org>\n\
-                    To: alice@example.org\n\
+    static MSGRMSG: &[u8] = b"From: Bob <bob@example.com>\n\
+                    To: alice@example.com\n\
                     Chat-Version: 1.0\n\
                     Subject: Chat: hello\n\
-                    Message-ID: <Mr.1111@example.org>\n\
+                    Message-ID: <Mr.1111@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:55 +0000\n\
                     \n\
                     hello\n";
 
-    static ONETOONE_NOREPLY_MAIL: &[u8] = b"From: Bob <bob@example.org>\n\
-                    To: alice@example.org\n\
+    static ONETOONE_NOREPLY_MAIL: &[u8] = b"From: Bob <bob@example.com>\n\
+                    To: alice@example.com\n\
                     Subject: Chat: hello\n\
-                    Message-ID: <2222@example.org>\n\
+                    Message-ID: <2222@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     hello\n";
 
-    static GRP_MAIL: &[u8] = b"From: bob@example.org\n\
-                    To: alice@example.org, claire@example.org\n\
+    static GRP_MAIL: &[u8] = b"From: bob@example.com\n\
+                    To: alice@example.com, claire@example.com\n\
                     Subject: group with Alice, Bob and Claire\n\
-                    Message-ID: <3333@example.org>\n\
+                    Message-ID: <3333@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                     \n\
                     hello\n";
 
     #[async_std::test]
     async fn test_adhoc_group_show_chats_only() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         assert_eq!(t.ctx.get_config_int(Config::ShowEmails).await, 0);
 
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
@@ -1907,7 +2015,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_accepted_contact_unknown() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("1"))
             .await
@@ -1923,12 +2031,12 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_accepted_contact_known() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("1"))
             .await
             .unwrap();
-        Contact::create(&t.ctx, "Bob", "bob@example.org")
+        Contact::create(&t.ctx, "Bob", "bob@example.com")
             .await
             .unwrap();
         dc_receive_imf(&t.ctx, GRP_MAIL, "INBOX", 1, false)
@@ -1943,7 +2051,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_accepted_contact_accepted() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("1"))
             .await
@@ -1989,7 +2097,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_all() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("2"))
             .await
@@ -2014,7 +2122,7 @@ mod tests {
     #[async_std::test]
     async fn test_read_receipt_and_unarchive() {
         // create alice's account
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
 
         // create one-to-one with bob, archive one-to-one
         let bob_id = Contact::create(&t.ctx, "bob", "bob@exampel.org")
@@ -2057,14 +2165,14 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             format!(
-                "From: alice@example.org\n\
-                 To: bob@example.org\n\
+                "From: alice@example.com\n\
+                 To: bob@example.com\n\
                  Subject: foo\n\
-                 Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Message-ID: <Gr.{}.12345678901@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Group-ID: {}\n\
                  Chat-Group-Name: foo\n\
-                 Chat-Disposition-Notification-To: alice@example.org\n\
+                 Chat-Disposition-Notification-To: alice@example.com\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                  \n\
                  hello\n",
@@ -2079,7 +2187,11 @@ mod tests {
         .unwrap();
         let msgs = chat::get_chat_msgs(&t.ctx, group_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
         let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
             .await
             .unwrap();
@@ -2093,12 +2205,12 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             format!(
-                "From: bob@example.org\n\
-                 To: alice@example.org\n\
+                "From: bob@example.com\n\
+                 To: alice@example.com\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
                  Chat-Version: 1.0\n\
-                 Message-ID: <Mr.12345678902@example.org>\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
                  Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
                  \n\
                  \n\
@@ -2112,9 +2224,9 @@ mod tests {
                  Content-Type: message/disposition-notification\n\
                  \n\
                  Reporting-UA: Delta Chat 1.28.0\n\
-                 Original-Recipient: rfc822;bob@example.org\n\
-                 Final-Recipient: rfc822;bob@example.org\n\
-                 Original-Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Original-Recipient: rfc822;bob@example.com\n\
+                 Final-Recipient: rfc822;bob@example.com\n\
+                 Original-Message-ID: <Gr.{}.12345678901@example.com>\n\
                  Disposition: manual-action/MDN-sent-automatically; displayed\n\
                  \n\
                  \n\
@@ -2154,7 +2266,7 @@ mod tests {
         // are very rare, however, we have to add them to the database (they go to the
         // "deaddrop" chat) to avoid a re-download from the server. See also [**]
 
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         let context = &t.ctx;
 
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
@@ -2162,9 +2274,9 @@ mod tests {
 
         dc_receive_imf(
             context,
-            b"To: bob@example.org\n\
+            b"To: bob@example.com\n\
                  Subject: foo\n\
-                 Message-ID: <3924@example.org>\n\
+                 Message-ID: <3924@example.com>\n\
                  Chat-Version: 1.0\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                  \n\
@@ -2183,7 +2295,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_escaped_from() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         let contact_id = Contact::create(&t.ctx, "foobar", "foobar@example.com")
             .await
             .unwrap();
@@ -2193,9 +2305,9 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
-                 To: alice@example.org\n\
+                 To: alice@example.com\n\
                  Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Message-ID: <asdklfjjaweofi@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Disposition-Notification-To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -2210,11 +2322,15 @@ mod tests {
                 .await
                 .unwrap()
                 .get_authname(),
-            "Фамилия Имя", // The name was "Имя, Фамилия" and ("lastname, firstname") and should be swapped to "firstname, lastname"
+            "Имя, Фамилия",
         );
         let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
         let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
             .await
             .unwrap();
@@ -2225,7 +2341,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_escaped_recipients() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         Contact::create(&t.ctx, "foobar", "foobar@example.com")
             .await
             .unwrap();
@@ -2239,10 +2355,10 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             b"From: Foobar <foobar@example.com>\n\
-                 To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.org\n\
+                 To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.com\n\
                  Cc: =?utf-8?q?=3Ch2=3E?= <carl@host.tld>\n\
                  Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Message-ID: <asdklfjjaweofi@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Disposition-Notification-To: <foobar@example.com>\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -2273,7 +2389,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_cc_to_contact() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         Contact::create(&t.ctx, "foobar", "foobar@example.com")
             .await
             .unwrap();
@@ -2291,10 +2407,10 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             b"From: Foobar <foobar@example.com>\n\
-                 To: alice@example.org\n\
+                 To: alice@example.com\n\
                  Cc: Carl <carl@host.tld>\n\
                  Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Message-ID: <asdklfjjaweofi@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Disposition-Notification-To: <foobar@example.com>\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -2314,4 +2430,187 @@ mod tests {
             "Carl"
         );
     }
+
+    #[async_std::test]
+    async fn test_parse_ndn_tiscali() {
+        test_parse_ndn(
+            "alice@tiscali.it",
+            "shenauithz@testrun.org",
+            "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
+            include_bytes!("../test-data/message/tiscali_ndn.eml"),
+            "",
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_testrun() {
+        test_parse_ndn(
+            "alice@testrun.org",
+            "hcksocnsofoejx@five.chat",
+            "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
+            include_bytes!("../test-data/message/testrun_ndn.eml"),
+            "Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_yahoo() {
+        test_parse_ndn(
+            "alice@yahoo.com",
+            "haeclirth.sinoenrat@yahoo.com",
+            "1680295672.3657931.1591783872936@mail.yahoo.com",
+            include_bytes!("../test-data/message/yahoo_ndn.eml"),
+            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_gmail() {
+        test_parse_ndn(
+            "alice@gmail.com",
+            "assidhfaaspocwaeofi@gmail.com",
+            "CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com",
+            include_bytes!("../test-data/message/gmail_ndn.eml"),
+            "Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp",
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_gmx() {
+        test_parse_ndn(
+            "alice@gmx.com",
+            "snaerituhaeirns@gmail.com",
+            "9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de",
+            include_bytes!("../test-data/message/gmx_ndn.eml"),
+            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_posteo() {
+        test_parse_ndn(
+            "alice@posteo.org",
+            "hanerthaertidiuea@gmx.de",
+            "04422840-f884-3e37-5778-8192fe22d8e1@posteo.de",
+            include_bytes!("../test-data/message/posteo_ndn.eml"),
+            "Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)",
+        )
+        .await;
+    }
+
+    // ndn = Non Delivery Notification
+    async fn test_parse_ndn(
+        self_addr: &str,
+        foreign_addr: &str,
+        rfc724_mid_outgoing: &str,
+        raw_ndn: &[u8],
+        error_msg: &str,
+    ) {
+        let t = TestContext::new().await;
+        t.configure_addr(self_addr).await;
+
+        dc_receive_imf(
+            &t.ctx,
+            format!(
+                "From: {}\n\
+                To: {}\n\
+                Subject: foo\n\
+                Message-ID: <{}>\n\
+                Chat-Version: 1.0\n\
+                Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                \n\
+                hello\n",
+                self_addr, foreign_addr, rfc724_mid_outgoing
+            )
+            .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+
+        // Check that the ndn would be downloaded:
+        let headers = mailparse::parse_mail(raw_ndn).unwrap().headers;
+        assert!(
+            crate::imap::prefetch_should_download(&t.ctx, &headers, ShowEmails::Off)
+                .await
+                .unwrap()
+        );
+
+        dc_receive_imf(&t.ctx, raw_ndn, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+
+        assert_eq!(msg.state, MessageState::OutFailed);
+        assert_eq!(msg.error, error_msg);
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_group_msg() {
+        let t = TestContext::new().await;
+        t.configure_addr("alice@gmail.com").await;
+
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@gmail.com\n\
+                 To: bob@example.com, assidhfaaspocwaeofi@gmail.com\n\
+                 Subject: foo\n\
+                 Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde\n\
+                 Chat-Group-Name: foo\n\
+                 Chat-Disposition-Notification-To: alice@example.com\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+
+        let raw = include_bytes!("../test-data/message/gmail_ndn_group.eml");
+        dc_receive_imf(&t.ctx, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+
+        assert_eq!(msg.state, MessageState::OutFailed);
+
+        let msgs = chat::get_chat_msgs(&t.ctx, msg.chat_id, 0, None).await;
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
+        let last_msg = Message::load_from_db(&t.ctx, *msg_id).await.unwrap();
+
+        assert_eq!(
+            last_msg.text,
+            Some(
+                t.ctx
+                    .stock_string_repl_str(
+                        StockMessage::FailedSendingTo,
+                        "assidhfaaspocwaeofi@gmail.com",
+                    )
+                    .await,
+            )
+        );
+        assert_eq!(last_msg.from_id, DC_CONTACT_ID_INFO);
+    }
 }

src/dc_tools.rs

@@ -482,15 +482,6 @@ pub struct InvalidEmailError {
     addr: String,
 }
 
-impl InvalidEmailError {
-    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
-        InvalidEmailError {
-            message: msg.into(),
-            addr: addr.into(),
-        }
-    }
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -530,17 +521,24 @@ impl FromStr for EmailAddress {
 
     /// Performs a dead-simple parse of an email address.
     fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        if input.is_empty() {
-            return Err(InvalidEmailError::new("empty string is not valid", input));
-        }
-        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
-
         let err = |msg: &str| {
             Err(InvalidEmailError {
                 message: msg.to_string(),
                 addr: input.to_string(),
             })
         };
+        if input.is_empty() {
+            return err("empty string is not valid");
+        }
+        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
+
+        if input
+            .chars()
+            .any(|c| c.is_whitespace() || c == '<' || c == '>')
+        {
+            return err("Email must not contain whitespaces, '>' or '<'");
+        }
+
         match &parts[..] {
             [domain, local] => {
                 if local.is_empty() {
@@ -776,7 +774,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_file_handling() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let context = &t.ctx;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
@@ -855,7 +853,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_ne!(
             dc_create_smeared_timestamp(&t.ctx).await,
             dc_create_smeared_timestamp(&t.ctx).await
@@ -871,7 +869,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamps() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
         let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
         let next = dc_smeared_time(&t.ctx).await;

src/e2ee.rs

@@ -11,7 +11,7 @@ use crate::context::Context;
 use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -87,30 +87,29 @@ impl EncryptHelper {
 
     /// Tries to encrypt the passed in `mail`.
     pub async fn encrypt(
-        &mut self,
+        self,
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: &[(Option<Peerstate<'_>>, &str)],
+        peerstates: Vec<(Option<Peerstate<'_>>, &str)>,
     ) -> Result<String> {
-        let mut keyring = Keyring::default();
+        let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
         for (peerstate, addr) in peerstates
-            .iter()
-            .filter_map(|(state, addr)| state.as_ref().map(|s| (s, addr)))
+            .into_iter()
+            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate.peek_key(min_verified).ok_or_else(|| {
+            let key = peerstate.take_key(min_verified).ok_or_else(|| {
                 format_err!("proper enc-key for {} missing, cannot encrypt", addr)
             })?;
-            keyring.add_ref(key);
+            keyring.add(key);
         }
-        let public_key = Key::from(self.public_key.clone());
-        keyring.add_ref(&public_key);
-        let sign_key = Key::from(SignedSecretKey::load_self(context).await?);
+        keyring.add(self.public_key.clone());
+        let sign_key = SignedSecretKey::load_self(context).await?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
-        let ctext = pgp::pk_encrypt(&raw_message, &keyring, Some(&sign_key))?;
+        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key)).await?;
 
         Ok(ctext)
     }
@@ -120,7 +119,7 @@ pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
@@ -151,41 +150,33 @@ pub async fn try_decrypt(
     }
 
     /* possibly perform decryption */
-    let mut private_keyring = Keyring::default();
-    let mut public_keyring_for_validate = Keyring::default();
-    let mut out_mail = None;
+    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
+    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
-    let self_addr = context.get_config(Config::ConfiguredAddr).await;
 
-    if let Some(self_addr) = self_addr {
-        if private_keyring
-            .load_self_private_for_decrypting(context, self_addr, &context.sql)
-            .await
-        {
-            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-                peerstate = Peerstate::from_addr(&context, &from).await;
-            }
-            if let Some(ref peerstate) = peerstate {
-                if peerstate.degrade_event.is_some() {
-                    handle_degrade_event(context, &peerstate).await?;
-                }
-                if let Some(ref key) = peerstate.gossip_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-                if let Some(ref key) = peerstate.public_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-            }
-
-            out_mail = decrypt_if_autocrypt_message(
-                context,
-                mail,
-                &private_keyring,
-                &public_keyring_for_validate,
-                &mut signatures,
-            )?;
+    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
+        peerstate = Peerstate::from_addr(&context, &from).await;
+    }
+    if let Some(peerstate) = peerstate {
+        if peerstate.degrade_event.is_some() {
+            handle_degrade_event(context, &peerstate).await?;
+        }
+        if let Some(key) = peerstate.gossip_key {
+            public_keyring_for_validate.add(key);
+        }
+        if let Some(key) = peerstate.public_key {
+            public_keyring_for_validate.add(key);
         }
     }
+
+    let out_mail = decrypt_if_autocrypt_message(
+        context,
+        mail,
+        private_keyring,
+        public_keyring_for_validate,
+        &mut signatures,
+    )
+    .await?;
     Ok((out_mail, signatures))
 }
 
@@ -216,12 +207,12 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
     Ok(&mail.subparts[1])
 }
 
-fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
     mail: &ParsedMail<'a>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     //  The returned bool is true if we detected an Autocrypt-encrypted
     // message and successfully decrypted it. Decryption then modifies the
@@ -240,21 +231,20 @@ fn decrypt_if_autocrypt_message<'a>(
     info!(context, "Detected Autocrypt-mime message");
 
     decrypt_part(
-        context,
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
         ret_valid_signatures,
     )
+    .await
 }
 
 /// Returns Ok(None) if nothing encrypted was found.
-fn decrypt_part(
-    _context: &Context,
+async fn decrypt_part(
     mail: &ParsedMail<'_>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
@@ -263,11 +253,12 @@ fn decrypt_part(
         ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
         let plain = pgp::pk_decrypt(
-            &data,
-            &private_keyring,
-            &public_keyring_for_validate,
+            data,
+            private_keyring,
+            public_keyring_for_validate,
             Some(ret_valid_signatures),
-        )?;
+        )
+        .await?;
 
         ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
         return Ok(Some(plain));
@@ -336,14 +327,14 @@ mod tests {
 
         #[async_std::test]
         async fn test_prexisting() {
-            let t = dummy_context().await;
-            let test_addr = configure_alice_keypair(&t.ctx).await;
+            let t = TestContext::new().await;
+            let test_addr = t.configure_alice().await;
             assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
-            let t = dummy_context().await;
+            let t = TestContext::new().await;
             assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
         }
     }

src/ephemeral.rs

@@ -0,0 +1,523 @@
+//! # Ephemeral messages
+//!
+//! Ephemeral messages are messages that have an Ephemeral-Timer
+//! header attached to them, which specifies time in seconds after
+//! which the message should be deleted both from the device and from
+//! the server. The timer is started when the message is marked as
+//! seen, which usually happens when its contents is displayed on
+//! device screen.
+//!
+//! Each chat, including 1:1, group chats and "saved messages" chat,
+//! has its own ephemeral timer setting, which is applied to all
+//! messages sent to the chat. The setting is synchronized to all the
+//! devices participating in the chat by applying the timer value from
+//! all received messages, including BCC-self ones, to the chat. This
+//! way the setting is eventually synchronized among all participants.
+//!
+//! When user changes ephemeral timer setting for the chat, a system
+//! message is automatically sent to update the setting for all
+//! participants. This allows changing the setting for a chat like any
+//! group chat setting, e.g. name and avatar, without the need to
+//! write an actual message.
+//!
+//! ## Device settings
+//!
+//! In addition to per-chat ephemeral message setting, each device has
+//! two global user-configured settings that complement per-chat
+//! settings: `delete_device_after` and `delete_server_after`. These
+//! settings are not synchronized among devices and apply to all
+//! messages known to the device, including messages sent or received
+//! before configuring the setting.
+//!
+//! `delete_device_after` configures the maximum time device is
+//! storing the messages locally. `delete_server_after` configures the
+//! time after which device will delete the messages it knows about
+//! from the server.
+//!
+//! ## How messages are deleted
+//!
+//! When the message is deleted locally, its contents is removed and
+//! it is moved to the trash chat. This database entry is then used to
+//! track the Message-ID and corresponding IMAP folder and UID until
+//! the message is deleted from the server. Vice versa, when device
+//! deletes the message from the server, it removes IMAP folder and
+//! UID information, but keeps the message contents. When database
+//! entry is both moved to trash chat and does not contain UID
+//! information, it is deleted from the database, leaving no trace of
+//! the message.
+//!
+//! ## When messages are deleted
+//!
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
+//!
+//! Server deletion happens by generating IMAP deletion jobs based on
+//! the database entries which are expired either according to their
+//! ephemeral message timers or global `delete_server_after` setting.
+
+use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use crate::constants::{
+    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::error::{ensure, Error};
+use crate::events::Event;
+use crate::message::{Message, MessageState, MsgId};
+use crate::mimeparser::SystemMessage;
+use crate::sql;
+use crate::stock::StockMessage;
+use async_std::task;
+use serde::{Deserialize, Serialize};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
+pub enum Timer {
+    Disabled,
+    Enabled { duration: u32 },
+}
+
+impl Timer {
+    pub fn to_u32(self) -> u32 {
+        match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => duration,
+        }
+    }
+
+    pub fn from_u32(duration: u32) -> Self {
+        if duration == 0 {
+            Self::Disabled
+        } else {
+            Self::Enabled { duration }
+        }
+    }
+}
+
+impl Default for Timer {
+    fn default() -> Self {
+        Self::Disabled
+    }
+}
+
+impl ToString for Timer {
+    fn to_string(&self) -> String {
+        self.to_u32().to_string()
+    }
+}
+
+impl FromStr for Timer {
+    type Err = ParseIntError;
+
+    fn from_str(input: &str) -> Result<Timer, ParseIntError> {
+        input.parse::<u32>().map(Self::from_u32)
+    }
+}
+
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
+    }
+}
+
+impl ChatId {
+    /// Get ephemeral message timer value in seconds.
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+        let timer = context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
+            )
+            .await?;
+        Ok(timer.unwrap_or_default())
+    }
+
+    /// Set ephemeral timer value without sending a message.
+    ///
+    /// Used when a message arrives indicating that someone else has
+    /// changed the timer value for a chat.
+    pub(crate) async fn inner_set_ephemeral_timer(
+        self,
+        context: &Context,
+        timer: Timer,
+    ) -> Result<(), Error> {
+        ensure!(!self.is_special(), "Invalid chat ID");
+
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+             SET ephemeral_timer=?
+             WHERE id=?;",
+                paramsv![timer, self],
+            )
+            .await?;
+        Ok(())
+    }
+
+    /// Set ephemeral message timer value in seconds.
+    ///
+    /// If timer value is 0, disable ephemeral message timer.
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+        if timer == self.get_ephemeral_timer(context).await? {
+            return Ok(());
+        }
+        self.inner_set_ephemeral_timer(context, timer).await?;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
+        if let Err(err) = send_msg(context, self, &mut msg).await {
+            error!(
+                context,
+                "Failed to send a message about ephemeral message timer change: {:?}", err
+            );
+        }
+        Ok(())
+    }
+}
+
+/// Returns a stock message saying that ephemeral timer is changed to `timer` by `from_id`.
+pub(crate) async fn stock_ephemeral_timer_changed(
+    context: &Context,
+    timer: Timer,
+    from_id: u32,
+) -> String {
+    let stock_message = match timer {
+        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+        Timer::Enabled { duration } => match duration {
+            60 => StockMessage::MsgEphemeralTimerMinute,
+            3600 => StockMessage::MsgEphemeralTimerHour,
+            86400 => StockMessage::MsgEphemeralTimerDay,
+            604_800 => StockMessage::MsgEphemeralTimerWeek,
+            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
+            _ => StockMessage::MsgEphemeralTimerEnabled,
+        },
+    };
+
+    context
+        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
+        .await
+}
+
+impl MsgId {
+    /// Returns ephemeral message timer value for the message.
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
+        let res = match context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
+            )
+            .await?
+        {
+            None | Some(0) => Timer::Disabled,
+            Some(duration) => Timer::Enabled { duration },
+        };
+        Ok(res)
+    }
+
+    /// Starts ephemeral message timer for the message if it is not started yet.
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
+        if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
+            let ephemeral_timestamp = time() + i64::from(duration);
+
+            context
+                .sql
+                .execute(
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
+                WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
+                AND id = ?",
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
+                )
+                .await?;
+            schedule_ephemeral_task(context).await;
+        }
+        Ok(())
+    }
+}
+
+/// Deletes messages which are expired according to
+/// `delete_device_after` setting or `ephemeral_timestamp` column.
+///
+/// Returns true if any message is deleted, so caller can emit
+/// MsgsChanged event. If nothing has been deleted, returns
+/// false. This function does not emit the MsgsChanged event itself,
+/// because it is also called when chatlist is reloaded, and emitting
+/// MsgsChanged there will cause infinite reload loop.
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+    let mut updated = context
+        .sql
+        .execute(
+            "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE \
+             ephemeral_timestamp != 0 \
+             AND ephemeral_timestamp < ? \
+             AND chat_id != ?",
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+        )
+        .await?
+        > 0;
+
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
+        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap_or_default()
+            .0;
+        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+            .await
+            .unwrap_or_default()
+            .0;
+
+        let threshold_timestamp = time() - delete_device_after;
+
+        // Delete expired messages
+        //
+        // Only update the rows that have to be updated, to avoid emitting
+        // unnecessary "chat modified" events.
+        let rows_modified = context
+            .sql
+            .execute(
+                "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE timestamp < ? \
+             AND chat_id > ? \
+             AND chat_id != ? \
+             AND chat_id != ?",
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
+            )
+            .await?;
+
+        updated |= rows_modified > 0;
+    }
+
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
+}
+
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
+///
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
+    let ephemeral_timestamp: Option<i64> = match context
+        .sql
+        .query_get_value_result(
+            "SELECT ephemeral_timestamp \
+         FROM msgs \
+         WHERE ephemeral_timestamp != 0 \
+           AND chat_id != ? \
+         ORDER BY ephemeral_timestamp ASC \
+         LIMIT 1",
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
+        )
+        .await
+    {
+        Err(err) => {
+            warn!(context, "Can't calculate next ephemeral timeout: {}", err);
+            return;
+        }
+        Ok(ephemeral_timestamp) => ephemeral_timestamp,
+    };
+
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
+
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+        let now = SystemTime::now();
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
+
+        if let Ok(duration) = until.duration_since(now) {
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                emit_event!(
+                    context1,
+                    Event::MsgsChanged {
+                        chat_id: ChatId::new(0),
+                        msg_id: MsgId::new(0)
+                    }
+                );
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            emit_event!(
+                context,
+                Event::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0)
+                }
+            );
+        }
+    }
+}
+
+/// Returns ID of any expired message that should be deleted from the server.
+///
+/// It looks up the trash chat too, to find messages that are already
+/// deleted locally, but not deleted on the server.
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+    let now = time();
+
+    let threshold_timestamp = match context.get_config_delete_server_after().await {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
+
+    context
+        .sql
+        .query_row_optional(
+            "SELECT id FROM msgs \
+         WHERE ( \
+         timestamp < ? \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         ) \
+         AND server_uid != 0 \
+         LIMIT 1",
+            paramsv![threshold_timestamp, now],
+            |row| row.get::<_, MsgId>(0),
+        )
+        .await
+}
+
+/// Start ephemeral timers for seen messages if they are not started
+/// yet.
+///
+/// It is possible that timers are not started due to a missing or
+/// failed `MsgId.start_ephemeral_timer()` call, either in the current
+/// or previous version of Delta Chat.
+///
+/// This function is supposed to be called in the background,
+/// e.g. from housekeeping task.
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+    context
+        .sql
+        .execute(
+            "UPDATE msgs \
+    SET ephemeral_timestamp = ? + ephemeral_timer \
+    WHERE ephemeral_timer > 0 \
+    AND ephemeral_timestamp = 0 \
+    AND state NOT IN (?, ?, ?)",
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
+        )
+        .await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::*;
+
+    #[async_std::test]
+    async fn test_stock_ephemeral_messages() {
+        let context = TestContext::new().await.ctx;
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            "Message deletion timer is disabled by me."
+        );
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
+            "Message deletion timer is disabled."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
+            "Message deletion timer is set to 1 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
+            "Message deletion timer is set to 30 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
+            "Message deletion timer is set to 1 minute."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
+            "Message deletion timer is set to 1 hour."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 day."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 week."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 4 * 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 4 weeks."
+        );
+    }
+}

src/events.rs

@@ -189,9 +189,16 @@ pub enum Event {
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// This event does not include ephemeral timer modification, which
+    /// is a separate event.
     #[strum(props(id = "2020"))]
     ChatModified(ChatId),
 
+    /// Chat ephemeral timer changed.
+    #[strum(props(id = "2021"))]
+    ChatEphemeralTimerModified { chat_id: ChatId, timer: u32 },
+
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.

src/headerdef.rs

@@ -21,6 +21,7 @@ pub enum HeaderDef {
     References,
     InReplyTo,
     Precedence,
+    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -41,6 +42,7 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    EphemeralTimer,
     _TestHeader,
 }
 

src/imap/client.rs

@@ -7,7 +7,7 @@ use async_imap::{
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
+use crate::login_param::dc_build_tls;
 
 use super::session::SessionStream;
 
@@ -78,10 +78,10 @@ impl Client {
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        certificate_checks: CertificateChecks,
+        strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
+        let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
@@ -118,16 +118,12 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
+    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {
             let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(certificate_checks);
+            let tls = dc_build_tls(strict_tls);
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();

src/imap/idle.rs

@@ -4,7 +4,7 @@ use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::context::Context;
+use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::select_folder;
 use super::session::Session;
@@ -13,19 +13,19 @@ type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
+    #[error("IMAP IDLE protocol failed to init/complete: {0}")]
     IdleProtocolFailed(#[from] async_imap::error::Error),
 
-    #[error("IMAP IDLE protocol timed out")]
+    #[error("IMAP IDLE protocol timed out: {0}")]
     IdleTimeout(#[from] async_std::future::TimeoutError),
 
     #[error("IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[error("IMAP select folder error")]
+    #[error("IMAP select folder error: {0}")]
     SelectFolderError(#[from] select_folder::Error),
 
-    #[error("Setup handle error")]
+    #[error("Setup handle error: {0}")]
     SetupHandleError(#[from] super::Error),
 }
 
@@ -34,7 +34,11 @@ impl Imap {
         self.config.can_idle
     }
 
-    pub async fn idle(&mut self, context: &Context, watch_folder: Option<String>) -> Result<()> {
+    pub async fn idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> Result<InterruptInfo> {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
@@ -44,10 +48,10 @@ impl Imap {
 
         self.select_folder(context, watch_folder.clone()).await?;
 
-        let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
+        let mut info = Default::default();
 
-        if let Some(session) = session {
+        if let Some(session) = self.session.take() {
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
                 return Err(Error::IdleProtocolFailed(err));
@@ -55,134 +59,129 @@ impl Imap {
 
             let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
 
-            if self.skip_next_idle_wait {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait = false;
-                drop(idle_wait);
+            enum Event {
+                IdleResponse(IdleResponse),
+                Interrupt(InterruptInfo),
+            }
+
+            info!(context, "Idle entering wait-on-remote state");
+            let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
+                let probe_network = self.idle_interrupt.recv().await;
+
+                // cancel imap idle connection properly
                 drop(interrupt);
 
-                info!(context, "Idle wait was skipped");
-            } else {
-                info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.race(
-                    self.idle_interrupt
-                        .recv()
-                        .map(|_| Ok(IdleResponse::ManualInterrupt)),
-                );
+                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+            });
 
-                match fut.await {
-                    Ok(IdleResponse::NewData(_)) => {
-                        info!(context, "Idle has NewData");
-                    }
-                    // TODO: idle_wait does not distinguish manual interrupts
-                    // from Timeouts if we would know it's a Timeout we could bail
-                    // directly and reconnect .
-                    Ok(IdleResponse::Timeout) => {
-                        info!(context, "Idle-wait timeout or interruption");
-                    }
-                    Ok(IdleResponse::ManualInterrupt) => {
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Err(err) => {
-                        warn!(context, "Idle wait errored: {:?}", err);
-                    }
+            match fut.await {
+                Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
+                    info!(context, "Idle has NewData");
+                }
+                Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                    info!(context, "Idle-wait timeout or interruption");
+                }
+                Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                    info!(context, "Idle wait was interrupted");
+                }
+                Ok(Event::Interrupt(i)) => {
+                    info = i;
+                    info!(context, "Idle wait was interrupted");
+                }
+                Err(err) => {
+                    warn!(context, "Idle wait errored: {:?}", err);
                 }
             }
 
-            // if we can't properly terminate the idle
-            // protocol let's break the connection.
-            let res = handle
+            let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(|err| {
-                    self.trigger_reconnect();
-                    Error::IdleTimeout(err)
-                })?;
-
-            match res {
-                Ok(session) => {
-                    self.session = Some(Session { inner: session });
-                }
-                Err(err) => {
-                    // if we cannot terminate IDLE it probably
-                    // means that we waited long (with idle_wait)
-                    // but the network went away/changed
-                    self.trigger_reconnect();
-                    return Err(Error::IdleProtocolFailed(err));
-                }
-            }
+                .map_err(Error::IdleTimeout)??;
+            self.session = Some(Session { inner: session });
+        } else {
+            warn!(context, "Attempted to idle without a session");
         }
 
-        Ok(())
+        Ok(info)
     }
 
-    pub(crate) async fn fake_idle(&mut self, context: &Context, watch_folder: Option<String>) {
+    pub(crate) async fn fake_idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> InterruptInfo {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
         let fake_idle_start_time = SystemTime::now();
         info!(context, "IMAP-fake-IDLEing...");
 
-        if self.skip_next_idle_wait {
-            // interrupt_idle has happened before we
-            // provided self.interrupt
-            self.skip_next_idle_wait = false;
-            info!(context, "fake-idle wait was skipped");
-        } else {
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        // Do not poll, just wait for an interrupt when no folder is passed in.
+        if watch_folder.is_none() {
+            return self.idle_interrupt.recv().await.unwrap_or_default();
+        }
 
-            // loop until we are interrupted or if we fetched something
-            loop {
-                use futures::future::FutureExt;
-                match interval
-                    .next()
-                    .race(self.idle_interrupt.recv().map(|_| None))
-                    .await
-                {
-                    Some(_) => {
-                        // try to connect with proper login params
-                        // (setup_handle_if_needed might not know about them if we
-                        // never successfully connected)
-                        if let Err(err) = self.connect_configured(context).await {
-                            warn!(context, "fake_idle: could not connect: {}", err);
-                            continue;
-                        }
-                        if self.config.can_idle {
-                            // we only fake-idled because network was gone during IDLE, probably
-                            break;
-                        }
-                        info!(context, "fake_idle is connected");
-                        // we are connected, let's see if fetching messages results
-                        // in anything.  If so, we behave as if IDLE had data but
-                        // will have already fetched the messages so perform_*_fetch
-                        // will not find any new.
+        // check every minute if there are new messages
+        // TODO: grow sleep durations / make them more flexible
+        let mut interval = async_std::stream::interval(Duration::from_secs(60));
 
-                        if let Some(ref watch_folder) = watch_folder {
-                            match self.fetch_new_messages(context, watch_folder).await {
-                                Ok(res) => {
-                                    info!(context, "fetch_new_messages returned {:?}", res);
-                                    if res {
-                                        break;
-                                    }
-                                }
-                                Err(err) => {
-                                    error!(context, "could not fetch from folder: {}", err);
-                                    self.trigger_reconnect()
+        enum Event {
+            Tick,
+            Interrupt(InterruptInfo),
+        }
+        // loop until we are interrupted or if we fetched something
+        let info = loop {
+            use futures::future::FutureExt;
+            match interval
+                .next()
+                .map(|_| Event::Tick)
+                .race(
+                    self.idle_interrupt
+                        .recv()
+                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                )
+                .await
+            {
+                Event::Tick => {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context).await {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break InterruptInfo::new(false, None);
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
+
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break InterruptInfo::new(false, None);
                                 }
                             }
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
                         }
                     }
-                    None => {
-                        // Interrupt
-                        break;
-                    }
+                }
+                Event::Interrupt(info) => {
+                    // Interrupt
+                    break info;
                 }
             }
-        }
+        };
 
         info!(
             context,
@@ -193,5 +192,7 @@ impl Imap {
                 .as_millis() as f64
                 / 1000.,
         );
+
+        info
     }
 }

src/imap/mod.rs

@@ -3,7 +3,9 @@
 //! uses [async-email/async-imap](https://github.com/async-email/async-imap)
 //! to implement connect, fetch, delete functionality with standard IMAP servers.
 
-use num_traits::FromPrimitive;
+#![forbid(clippy::indexing_slicing)]
+
+use std::collections::BTreeMap;
 
 use async_imap::{
     error::Result as ImapResult,
@@ -11,6 +13,7 @@ use async_imap::{
 };
 use async_std::prelude::*;
 use async_std::sync::Receiver;
+use num_traits::FromPrimitive;
 
 use crate::config::*;
 use crate::constants::*;
@@ -26,7 +29,8 @@ use crate::message::{self, update_server_uid};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
-use crate::stock::StockMessage;
+use crate::provider::get_provider_info;
+use crate::{scheduler::InterruptInfo, stock::StockMessage};
 
 mod client;
 mod idle;
@@ -108,12 +112,11 @@ const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
-    idle_interrupt: Receiver<()>,
+    idle_interrupt: Receiver<InterruptInfo>,
     config: ImapConfig,
     session: Option<Session>,
     connected: bool,
     interrupt: Option<stop_token::StopSource>,
-    skip_next_idle_wait: bool,
     should_reconnect: bool,
 }
 
@@ -148,7 +151,7 @@ struct ImapConfig {
     pub imap_port: u16,
     pub imap_user: String,
     pub imap_pw: String,
-    pub certificate_checks: CertificateChecks,
+    pub strict_tls: bool,
     pub server_flags: usize,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
@@ -168,7 +171,7 @@ impl Default for ImapConfig {
             imap_port: 0,
             imap_user: "".into(),
             imap_pw: "".into(),
-            certificate_checks: Default::default(),
+            strict_tls: false,
             server_flags: 0,
             selected_folder: None,
             selected_mailbox: None,
@@ -180,14 +183,13 @@ impl Default for ImapConfig {
 }
 
 impl Imap {
-    pub fn new(idle_interrupt: Receiver<()>) -> Self {
+    pub fn new(idle_interrupt: Receiver<InterruptInfo>) -> Self {
         Imap {
             idle_interrupt,
             config: Default::default(),
             session: Default::default(),
             connected: Default::default(),
             interrupt: Default::default(),
-            skip_next_idle_wait: Default::default(),
             should_reconnect: Default::default(),
         }
     }
@@ -227,7 +229,7 @@ impl Imap {
                 match Client::connect_insecure((imap_server, imap_port)).await {
                     Ok(client) => {
                         if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.certificate_checks).await
+                            client.secure(imap_server, config.strict_tls).await
                         } else {
                             Ok(client)
                         }
@@ -239,12 +241,8 @@ impl Imap {
                 let imap_server: &str = config.imap_server.as_ref();
                 let imap_port = config.imap_port;
 
-                Client::connect_secure(
-                    (imap_server, imap_port),
-                    imap_server,
-                    config.certificate_checks,
-                )
-                .await
+                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
+                    .await
             };
 
         let login_res = match connection_res {
@@ -294,6 +292,8 @@ impl Imap {
 
         match login_res {
             Ok(session) => {
+                // needs to be set here to ensure it is set on reconnects.
+                self.connected = true;
                 self.session = Some(session);
                 Ok(())
             }
@@ -314,9 +314,15 @@ impl Imap {
     }
 
     async fn unsetup_handle(&mut self, context: &Context) {
+        // Close folder if messages should be expunged
+        if let Err(err) = self.close_folder(context).await {
+            warn!(context, "failed to close folder: {:?}", err);
+        }
+
+        // Logout from the server
         if let Some(mut session) = self.session.take() {
-            if let Err(err) = session.close().await {
-                warn!(context, "failed to close connection: {:?}", err);
+            if let Err(err) = session.logout().await {
+                warn!(context, "failed to logout: {:?}", err);
             }
         }
         self.connected = false;
@@ -376,7 +382,15 @@ impl Imap {
             config.imap_port = imap_port;
             config.imap_user = imap_user.to_string();
             config.imap_pw = imap_pw.to_string();
-            config.certificate_checks = lp.imap_certificate_checks;
+            let provider = get_provider_info(&lp.addr);
+            config.strict_tls = match lp.imap_certificate_checks {
+                CertificateChecks::Automatic => {
+                    provider.map_or(false, |provider| provider.strict_tls)
+                }
+                CertificateChecks::Strict => true,
+                CertificateChecks::AcceptInvalidCertificates
+                | CertificateChecks::AcceptInvalidCertificates2 => false,
+            };
             config.server_flags = server_flags;
         }
 
@@ -577,133 +591,120 @@ impl Imap {
             .select_with_uidvalidity(context, folder.as_ref())
             .await?;
 
-        let mut read_cnt: usize = 0;
+        let msgs = self.fetch_after(context, last_seen_uid).await?;
+        let read_cnt = msgs.len();
+        let folder: &str = folder.as_ref();
+
         let mut read_errors = 0;
+        let mut uids = Vec::with_capacity(msgs.len());
+        let mut new_last_seen_uid = None;
 
-        // prefetch info from all unfetched mails
-        let mut new_last_seen_uid = last_seen_uid;
-
-        if self.session.is_none() {
-            return Err(Error::NoConnection);
-        }
-        let session = self.session.as_mut().unwrap();
-
-        // fetch messages with larger UID than the last one seen
-        // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
-        let set = format!("{}:*", last_seen_uid + 1);
-        let mut list = match session.uid_fetch(set, PREFETCH_FLAGS).await {
-            Ok(list) => list,
-            Err(err) => {
-                return Err(Error::FetchFailed(err));
-            }
-        };
-
-        let mut msgs = Vec::new();
-        while let Some(fetch) = list.next().await {
-            let fetch = fetch.map_err(|err| Error::Other(err.to_string()))?;
-            msgs.push(fetch);
-        }
-        drop(list);
-
-        msgs.sort_unstable_by_key(|msg| msg.uid.unwrap_or_default());
-
-        for fetch in msgs.into_iter() {
-            let cur_uid = fetch.uid.unwrap_or_default();
-            if cur_uid <= last_seen_uid {
-                // If the mailbox is not empty, results always include
-                // at least one UID, even if last_seen_uid+1 is past
-                // the last UID in the mailbox.  It happens because
-                // uid+1:* is interpreted the same way as *:uid+1.
-                // See https://tools.ietf.org/html/rfc3501#page-61 for
-                // standard reference. Therefore, sometimes we receive
-                // already seen messages and have to filter them out.
-                info!(
-                    context,
-                    "fetch_new_messages: ignoring uid {}, last seen was {}", cur_uid, last_seen_uid
-                );
-                continue;
-            }
-            read_cnt += 1;
-            let headers = get_fetch_headers(&fetch)?;
-            let message_id = prefetch_get_message_id(&headers).unwrap_or_default();
-            if let Ok(true) = precheck_imf(context, &message_id, folder.as_ref(), cur_uid)
-                .await
-                .map_err(|err| {
-                    warn!(context, "precheck_imf error: {}", err);
-                    err
-                })
-            {
-                // we know the message-id already or don't want the message otherwise.
-                info!(
-                    context,
-                    "Skipping message {} from \"{}\" by precheck.",
-                    message_id,
-                    folder.as_ref(),
-                );
-            } else {
-                // we do not know the message-id
-                // or the message-id is missing (in this case, we create one in the further process)
-                // or some other error happened
-                let show = prefetch_should_download(context, &headers, show_emails)
-                    .await
-                    .map_err(|err| {
-                        warn!(context, "prefetch_should_download error: {}", err);
-                        err
-                    })
-                    .unwrap_or(true);
-
-                if !show {
-                    info!(
-                        context,
-                        "Ignoring new message {} from \"{}\".",
-                        message_id,
-                        folder.as_ref(),
-                    );
-                } else {
-                    // check passed, go fetch the rest
-                    if let Err(err) = self.fetch_single_msg(context, &folder, cur_uid).await {
-                        info!(
-                            context,
-                            "Read error for message {} from \"{}\", trying over later: {}.",
-                            message_id,
-                            folder.as_ref(),
-                            err
-                        );
-                        read_errors += 1;
-                    }
+        for (current_uid, msg) in msgs.into_iter() {
+            let (headers, msg_id) = match get_fetch_headers(&msg) {
+                Ok(headers) => {
+                    let msg_id = prefetch_get_message_id(&headers).unwrap_or_default();
+                    (headers, msg_id)
                 }
-            }
+                Err(err) => {
+                    warn!(context, "{}", err);
+                    read_errors += 1;
+                    continue;
+                }
+            };
 
-            if read_errors == 0 {
-                new_last_seen_uid = cur_uid;
+            if message_needs_processing(
+                context,
+                current_uid,
+                &headers,
+                &msg_id,
+                folder,
+                show_emails,
+            )
+            .await
+            {
+                // Trigger download and processing for this message.
+                uids.push(current_uid);
+            } else if read_errors == 0 {
+                // No errors so far, but this was skipped, so mark as last_seen_uid
+                new_last_seen_uid = Some(current_uid);
             }
         }
 
-        if new_last_seen_uid > last_seen_uid {
-            self.set_config_last_seen_uid(context, &folder, uid_validity, new_last_seen_uid)
+        // check passed, go fetch the emails
+        let (new_last_seen_uid_processed, error_cnt) =
+            self.fetch_many_msgs(context, &folder, &uids).await;
+        read_errors += error_cnt;
+
+        // determine which last_seen_uid to use to update  to
+        let new_last_seen_uid_processed = new_last_seen_uid_processed.unwrap_or_default();
+        let new_last_seen_uid = new_last_seen_uid.unwrap_or_default();
+        let last_one = new_last_seen_uid.max(new_last_seen_uid_processed);
+
+        if last_one > last_seen_uid {
+            self.set_config_last_seen_uid(context, &folder, uid_validity, last_one)
                 .await;
         }
 
-        if read_errors > 0 {
+        if read_errors == 0 {
+            info!(context, "{} mails read from \"{}\".", read_cnt, folder,);
+        } else {
             warn!(
                 context,
-                "{} mails read from \"{}\" with {} errors.",
-                read_cnt,
-                folder.as_ref(),
-                read_errors
-            );
-        } else {
-            info!(
-                context,
-                "{} mails read from \"{}\".",
-                read_cnt,
-                folder.as_ref()
+                "{} mails read from \"{}\" with {} errors.", read_cnt, folder, read_errors
             );
         }
 
         Ok(read_cnt > 0)
     }
 
+    /// Fetch all uids larger than the passed in. Returns a sorted list of fetch results.
+    async fn fetch_after(
+        &mut self,
+        context: &Context,
+        uid: u32,
+    ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
+        if self.session.is_none() {
+            return Err(Error::NoConnection);
+        }
+
+        let session = self.session.as_mut().unwrap();
+
+        // fetch messages with larger UID than the last one seen
+        // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
+        let set = format!("{}:*", uid + 1);
+        let mut list = session
+            .uid_fetch(set, PREFETCH_FLAGS)
+            .await
+            .map_err(Error::FetchFailed)?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(fetch) = list.next().await {
+            let msg = fetch.map_err(|err| Error::Other(err.to_string()))?;
+            if let Some(msg_uid) = msg.uid {
+                msgs.insert(msg_uid, msg);
+            }
+        }
+        drop(list);
+
+        // If the mailbox is not empty, results always include
+        // at least one UID, even if last_seen_uid+1 is past
+        // the last UID in the mailbox.  It happens because
+        // uid+1:* is interpreted the same way as *:uid+1.
+        // See https://tools.ietf.org/html/rfc3501#page-61 for
+        // standard reference. Therefore, sometimes we receive
+        // already seen messages and have to filter them out.
+        let new_msgs = msgs.split_off(&(uid + 1));
+
+        for current_uid in msgs.keys() {
+            info!(
+                context,
+                "fetch_new_messages: ignoring uid {}, last seen was {}", current_uid, uid
+            );
+        }
+
+        Ok(new_msgs)
+    }
+
     async fn set_config_last_seen_uid<S: AsRef<str>>(
         &self,
         context: &Context,
@@ -721,69 +722,107 @@ impl Imap {
             .ok();
     }
 
-    /// Fetches a single message by server UID.
+    /// Fetches a list of messages by server UID.
+    /// The passed in list of uids must be sorted.
     ///
-    /// If it succeeds, the message should be treated as received even
-    /// if no database entries are created. If the function returns an
-    /// error, the caller should try again later.
-    async fn fetch_single_msg<S: AsRef<str>>(
+    /// Returns the last uid fetch successfully and an error count.
+    async fn fetch_many_msgs<S: AsRef<str>>(
         &mut self,
         context: &Context,
         folder: S,
-        server_uid: u32,
-    ) -> Result<()> {
-        if !self.is_connected() {
-            return Err(Error::Other("Not connected".to_string()));
-        }
-
-        let set = format!("{}", server_uid);
-
-        let mut msgs = if let Some(ref mut session) = &mut self.session {
-            match session.uid_fetch(set, BODY_FLAGS).await {
-                Ok(msgs) => msgs,
-                Err(err) => {
-                    // TODO maybe differentiate between IO and input/parsing problems
-                    // so we don't reconnect if we have a (rare) input/output parsing problem?
-                    self.should_reconnect = true;
-                    warn!(
-                        context,
-                        "Error on fetching message #{} from folder \"{}\"; error={}.",
-                        server_uid,
-                        folder.as_ref(),
-                        err
-                    );
-                    return Err(Error::FetchFailed(err));
-                }
+        server_uids: &[u32],
+    ) -> (Option<u32>, usize) {
+        let set = match server_uids {
+            [] => return (None, 0),
+            [server_uid] => server_uid.to_string(),
+            [first_uid, .., last_uid] => {
+                // XXX: it is assumed that UIDs are sorted and
+                // contiguous. If UIDs are not contiguous, more
+                // messages than needed will be downloaded.
+                debug_assert!(first_uid < last_uid, "uids must be sorted");
+                format!("{}:{}", first_uid, last_uid)
             }
-        } else {
-            // we could not get a valid imap session, this should be retried
-            self.trigger_reconnect();
-            return Err(Error::Other("Could not get IMAP session".to_string()));
         };
 
-        if let Some(Ok(msg)) = msgs.next().await {
-            // XXX put flags into a set and pass them to dc_receive_imf
+        if !self.is_connected() {
+            warn!(context, "Not connected");
+            return (None, server_uids.len());
+        }
+
+        if self.session.is_none() {
+            // we could not get a valid imap session, this should be retried
+            self.trigger_reconnect();
+            warn!(context, "Could not get IMAP session");
+            return (None, server_uids.len());
+        }
+
+        let session = self.session.as_mut().unwrap();
+
+        let mut msgs = match session.uid_fetch(&set, BODY_FLAGS).await {
+            Ok(msgs) => msgs,
+            Err(err) => {
+                // TODO: maybe differentiate between IO and input/parsing problems
+                // so we don't reconnect if we have a (rare) input/output parsing problem?
+                self.should_reconnect = true;
+                warn!(
+                    context,
+                    "Error on fetching messages #{} from folder \"{}\"; error={}.",
+                    &set,
+                    folder.as_ref(),
+                    err
+                );
+                return (None, server_uids.len());
+            }
+        };
+
+        let folder = folder.as_ref().to_string();
+
+        let mut read_errors = 0;
+        let mut last_uid = None;
+        let mut count = 0;
+
+        while let Some(Ok(msg)) = msgs.next().await {
+            let server_uid = msg.uid.unwrap_or_default();
+
+            if !server_uids.contains(&server_uid) {
+                // skip if there are some in between we are not interested in
+                continue;
+            }
+            count += 1;
+
             let is_deleted = msg.flags().any(|flag| flag == Flag::Deleted);
+            if is_deleted || msg.body().is_none() {
+                // No need to process these.
+                continue;
+            }
+
+            // XXX put flags into a set and pass them to dc_receive_imf
+            let context = context.clone();
+            let folder = folder.clone();
+
+            // safe, as we checked above that there is a body.
+            let body = msg.body().unwrap();
             let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
-            if !is_deleted && msg.body().is_some() {
-                let body = msg.body().unwrap_or_default();
-                if let Err(err) =
-                    dc_receive_imf(context, &body, folder.as_ref(), server_uid, is_seen).await
-                {
-                    return Err(Error::Other(format!("dc_receive_imf error: {}", err)));
+            match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
+                Ok(_) => last_uid = Some(server_uid),
+                Err(err) => {
+                    warn!(context, "dc_receive_imf error: {}", err);
+                    read_errors += 1;
                 }
-            }
-        } else {
+            };
+        }
+
+        if count != server_uids.len() {
             warn!(
                 context,
-                "Message #{} does not exist in folder \"{}\".",
-                server_uid,
-                folder.as_ref()
+                "failed to fetch all uids: got {}, requested {}",
+                count,
+                server_uids.len()
             );
         }
 
-        Ok(())
+        (last_uid, read_errors)
     }
 
     pub async fn can_move(&self) -> bool {
@@ -929,7 +968,7 @@ impl Imap {
         uid: u32,
     ) -> Option<ImapActionResult> {
         if uid == 0 {
-            return Some(ImapActionResult::Failed);
+            return Some(ImapActionResult::RetryLater);
         }
         if !self.is_connected() {
             // currently jobs are only performed on the INBOX thread
@@ -1097,54 +1136,54 @@ impl Imap {
                 }
             };
 
+            let mut delimiter = ".".to_string();
+            let mut delimiter_is_default = true;
             let mut sentbox_folder = None;
             let mut mvbox_folder = None;
-
-            let mut delimiter = ".".to_string();
-            if let Some(folder) = folders.next().await {
-                let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
-                if let Some(d) = folder.delimiter() {
-                    if !d.is_empty() {
-                        delimiter = d.to_string();
-                    }
-                }
-            }
-            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
-            let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
+            let mut fallback_folder = get_fallback_folder(&delimiter);
 
             while let Some(folder) = folders.next().await {
                 let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
                 info!(context, "Scanning folder: {:?}", folder);
 
-                if mvbox_folder.is_none()
-                    && (folder.name() == "DeltaChat" || folder.name() == fallback_folder)
-                {
-                    mvbox_folder = Some(folder.name().to_string());
-                }
-
-                if sentbox_folder.is_none() {
-                    if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
-                        sentbox_folder = Some(folder);
-                    } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
-                        sentbox_folder = Some(folder);
+                // Update the delimiter iff there is a different one, but only once.
+                if let Some(d) = folder.delimiter() {
+                    if delimiter_is_default && !d.is_empty() && delimiter != d {
+                        delimiter = d.to_string();
+                        fallback_folder = get_fallback_folder(&delimiter);
+                        delimiter_is_default = false;
                     }
                 }
 
-                if mvbox_folder.is_some() && sentbox_folder.is_some() {
-                    break;
+                if folder.name() == "DeltaChat" {
+                    // Always takes precendent
+                    mvbox_folder = Some(folder.name().to_string());
+                } else if folder.name() == fallback_folder {
+                    // only set iff none has been already set
+                    if mvbox_folder.is_none() {
+                        mvbox_folder = Some(folder.name().to_string());
+                    }
+                } else if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
+                    // Always takes precedent
+                    sentbox_folder = Some(folder.name().to_string());
+                } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
+                    // only set iff none has been already set
+                    if sentbox_folder.is_none() {
+                        sentbox_folder = Some(folder.name().to_string());
+                    }
                 }
             }
-            info!(context, "sentbox folder is {:?}", sentbox_folder);
-
             drop(folders);
 
+            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
+            info!(context, "sentbox folder is {:?}", sentbox_folder);
+
             if mvbox_folder.is_none() && create_mvbox {
                 info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
 
                 match session.create("DeltaChat").await {
                     Ok(_) => {
                         mvbox_folder = Some("DeltaChat".into());
-
                         info!(context, "MVBOX-folder created.",);
                     }
                     Err(err) => {
@@ -1178,23 +1217,16 @@ impl Imap {
                 }
             }
             context
-                .sql
-                .set_raw_config(context, "configured_inbox_folder", Some("INBOX"))
+                .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
                 .await?;
             if let Some(ref mvbox_folder) = mvbox_folder {
                 context
-                    .sql
-                    .set_raw_config(context, "configured_mvbox_folder", Some(mvbox_folder))
+                    .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
                     .await?;
             }
             if let Some(ref sentbox_folder) = sentbox_folder {
                 context
-                    .sql
-                    .set_raw_config(
-                        context,
-                        "configured_sentbox_folder",
-                        Some(sentbox_folder.name()),
-                    )
+                    .set_config(Config::ConfiguredSentboxFolder, Some(sentbox_folder))
                     .await?;
             }
             context
@@ -1352,7 +1384,11 @@ async fn precheck_imf(
         }
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
-            update_server_uid(context, &rfc724_mid, server_folder, server_uid).await;
+            update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
+            context
+                .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
+                .await;
+            info!(context, "Updating server_uid and interrupting")
         }
         Ok(true)
     } else {
@@ -1396,7 +1432,7 @@ async fn prefetch_is_reply_to_chat_message(
     false
 }
 
-async fn prefetch_should_download(
+pub(crate) async fn prefetch_should_download(
     context: &Context,
     headers: &[mailparse::MailHeader<'_>],
     show_emails: ShowEmails,
@@ -1404,6 +1440,13 @@ async fn prefetch_should_download(
     let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
     let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await;
 
+    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
+        let from = from.to_ascii_lowercase();
+        from.contains("mailer-daemon") || from.contains("mail-daemon")
+    } else {
+        false
+    };
+
     // Autocrypt Setup Message should be shown even if it is from non-chat client.
     let is_autocrypt_setup_message = headers
         .get_header_value(HeaderDef::AutocryptSetupMessage)
@@ -1414,6 +1457,7 @@ async fn prefetch_should_download(
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message
+        || maybe_ndn
         || match show_emails {
             ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
             ShowEmails::AcceptedContacts => {
@@ -1424,3 +1468,54 @@ async fn prefetch_should_download(
     let show = show && !blocked_contact;
     Ok(show)
 }
+
+async fn message_needs_processing(
+    context: &Context,
+    current_uid: u32,
+    headers: &[mailparse::MailHeader<'_>],
+    msg_id: &str,
+    folder: &str,
+    show_emails: ShowEmails,
+) -> bool {
+    let skip = match precheck_imf(context, &msg_id, folder, current_uid).await {
+        Ok(skip) => skip,
+        Err(err) => {
+            warn!(context, "precheck_imf error: {}", err);
+            true
+        }
+    };
+
+    if skip {
+        // we know the message-id already or don't want the message otherwise.
+        info!(
+            context,
+            "Skipping message {} from \"{}\" by precheck.", msg_id, folder,
+        );
+        return false;
+    }
+
+    // we do not know the message-id
+    // or the message-id is missing (in this case, we create one in the further process)
+    // or some other error happened
+    let show = match prefetch_should_download(context, &headers, show_emails).await {
+        Ok(show) => show,
+        Err(err) => {
+            warn!(context, "prefetch_should_download error: {}", err);
+            true
+        }
+    };
+
+    if !show {
+        info!(
+            context,
+            "Ignoring new message {} from \"{}\".", msg_id, folder,
+        );
+        return false;
+    }
+
+    true
+}
+
+fn get_fallback_folder(delimiter: &str) -> String {
+    format!("INBOX{}DeltaChat", delimiter)
+}

src/imap/select_folder.rs

@@ -27,7 +27,7 @@ impl Imap {
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 

src/imex.rs

@@ -1,5 +1,6 @@
 //! # Import/export module
 
+use std::any::Any;
 use std::cmp::{max, min};
 
 use async_std::path::{Path, PathBuf};
@@ -16,7 +17,7 @@ use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
-use crate::key::{self, DcKey, Key, SignedSecretKey};
+use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -181,13 +182,13 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let private_key = Key::from(SignedSecretKey::load_self(context).await?);
+    let private_key = SignedSecretKey::load_self(context).await?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
+    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -274,7 +275,7 @@ pub async fn continue_key_transfer(
     if let Some(filename) = msg.get_file(context) {
         let file = dc_open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
-        let armored_key = decrypt_setup_file(context, &sc, file)?;
+        let armored_key = decrypt_setup_file(&sc, file).await?;
         set_self_key(context, &armored_key, true, true).await?;
         maybe_add_bcc_self_device_msg(context).await?;
 
@@ -291,12 +292,8 @@ async fn set_self_key(
     prefer_encrypt_required: bool,
 ) -> Result<()> {
     // try hard to only modify key-state
-    let keys = Key::from_armored_string(armored, KeyType::Private)
-        .and_then(|(k, h)| if k.verify() { Some((k, h)) } else { None })
-        .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
-
-    ensure!(keys.is_some(), "Not a valid private key");
-    let (private_key, public_key, header) = keys.unwrap();
+    let (private_key, header) = SignedSecretKey::from_asc(armored)?;
+    let public_key = private_key.split_public_key()?;
     let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
     match preferencrypt.map(|s| s.as_str()) {
         Some(headerval) => {
@@ -322,15 +319,10 @@ async fn set_self_key(
     let self_addr = context.get_config(Config::ConfiguredAddr).await;
     ensure!(self_addr.is_some(), "Missing self addr");
     let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
-
-    let (public, secret) = match (public_key, private_key) {
-        (Key::Public(p), Key::Secret(s)) => (p, s),
-        _ => bail!("wrong keys unpacked"),
-    };
     let keypair = pgp::KeyPair {
         addr,
-        public,
-        secret,
+        public: public_key,
+        secret: private_key,
     };
     key::store_self_keypair(
         context,
@@ -345,12 +337,11 @@ async fn set_self_key(
     Ok(())
 }
 
-fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
-    _context: &Context,
+async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     file: T,
 ) -> Result<String> {
-    let plain_bytes = pgp::symm_decrypt(passphrase, file)?;
+    let plain_bytes = pgp::symm_decrypt(passphrase, file).await?;
     let plain_text = std::string::String::from_utf8(plain_bytes)?;
 
     Ok(plain_text)
@@ -697,9 +688,9 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
             |row| {
                 let id = row.get(0)?;
                 let public_key_blob: Vec<u8> = row.get(1)?;
-                let public_key = Key::from_slice(&public_key_blob, KeyType::Public);
+                let public_key = SignedPublicKey::from_slice(&public_key_blob);
                 let private_key_blob: Vec<u8> = row.get(2)?;
-                let private_key = Key::from_slice(&private_key_blob, KeyType::Private);
+                let private_key = SignedSecretKey::from_slice(&private_key_blob);
                 let is_default: i32 = row.get(3)?;
 
                 Ok((id, public_key, private_key, is_default))
@@ -713,7 +704,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
 
     for (id, public_key, private_key, is_default) in keys {
         let id = Some(id).filter(|_| is_default != 0);
-        if let Some(key) = public_key {
+        if let Ok(key) = public_key {
             if export_key_to_asc_file(context, &dir, id, &key)
                 .await
                 .is_err()
@@ -723,7 +714,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
         } else {
             export_errors += 1;
         }
-        if let Some(key) = private_key {
+        if let Ok(key) = private_key {
             if export_key_to_asc_file(context, &dir, id, &key)
                 .await
                 .is_err()
@@ -742,22 +733,32 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
 /*******************************************************************************
  * Classic key export
  ******************************************************************************/
-async fn export_key_to_asc_file(
+async fn export_key_to_asc_file<T>(
     context: &Context,
     dir: impl AsRef<Path>,
     id: Option<i64>,
-    key: &Key,
-) -> std::io::Result<()> {
+    key: &T,
+) -> std::io::Result<()>
+where
+    T: DcKey + Any,
+{
     let file_name = {
-        let kind = if key.is_public() { "public" } else { "private" };
+        let any_key = key as &dyn Any;
+        let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "public"
+        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "private"
+        } else {
+            "unknown"
+        };
         let id = id.map_or("default".into(), |i| i.to_string());
-
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
     info!(context, "Exporting key {}", file_name.display());
     dc_delete_file(context, &file_name).await;
 
-    let res = key.write_asc_to_file(&file_name, context).await;
+    let content = key.to_asc(None).into_bytes();
+    let res = dc_write_file(context, &file_name, &content).await;
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
@@ -775,9 +776,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
@@ -794,12 +795,12 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
@@ -807,7 +808,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_setup_code() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let setupcode = create_setup_code(&t.ctx);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
@@ -822,8 +823,8 @@ mod tests {
 
     #[async_std::test]
     async fn test_export_key_to_asc_file() {
-        let context = dummy_context().await;
-        let key = Key::from(alice_keypair().public);
+        let context = TestContext::new().await;
+        let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
             .await
@@ -852,9 +853,6 @@ mod tests {
 
     #[async_std::test]
     async fn test_split_and_decrypt() {
-        let ctx = dummy_context().await;
-        let context = &ctx.ctx;
-
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
         let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
         assert_eq!(typ, BlockType::Message);
@@ -864,12 +862,10 @@ mod tests {
         assert!(!base64.is_empty());
 
         let setup_file = S_EM_SETUPFILE.to_string();
-        let decrypted = decrypt_setup_file(
-            context,
-            S_EM_SETUPCODE,
-            std::io::Cursor::new(setup_file.as_bytes()),
-        )
-        .unwrap();
+        let decrypted =
+            decrypt_setup_file(S_EM_SETUPCODE, std::io::Cursor::new(setup_file.as_bytes()))
+                .await
+                .unwrap();
 
         let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
 

src/job.rs

@@ -21,6 +21,7 @@ use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::load_imap_deletion_msgid;
 use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::Event;
 use crate::imap::*;
@@ -31,7 +32,7 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
-use crate::sql;
+use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
@@ -190,7 +191,7 @@ impl Job {
     /// Saves the job to the database, creating a new entry if necessary.
     ///
     /// The Job is consumed by this method.
-    pub async fn save(self, context: &Context) -> Result<()> {
+    pub(crate) async fn save(self, context: &Context) -> Result<()> {
         let thread: Thread = self.action.into();
 
         info!(context, "saving job for {}-thread: {:?}", thread, self);
@@ -329,7 +330,7 @@ impl Job {
         }
     }
 
-    pub async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
         if !smtp.is_connected().await {
             let loginparam = LoginParam::from_database(context, "configured_").await;
@@ -498,16 +499,13 @@ impl Job {
     }
 
     async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-
-        if let Err(err) = imap.ensure_configured_folders(context, true).await {
-            warn!(context, "could not configure folders: {:?}", err);
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
-        let dest_folder = context
-            .sql
-            .get_raw_config(context, "configured_mvbox_folder")
-            .await;
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
 
         if let Some(dest_folder) = dest_folder {
             let server_folder = msg.server_folder.as_ref().unwrap();
@@ -518,7 +516,7 @@ impl Job {
             {
                 ImapActionResult::RetryLater => Status::RetryLater,
                 ImapActionResult::Success => {
-                    // XXX Rust-Imap provides no target uid on mv, so just set it to 0
+                    // Rust-Imap provides no target uid on mv, so just set it to 0, update again when precheck_imf() is called for the moved message
                     message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await;
                     Status::Finished(Ok(()))
                 }
@@ -541,6 +539,11 @@ impl Job {
     /// records pointing to the same message on the server, the job
     /// also removes the message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         if !msg.rfc724_mid.is_empty() {
@@ -611,12 +614,13 @@ impl Job {
     }
 
     async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = context
-                .sql
-                .get_raw_config(context, "configured_mvbox_folder")
-                .await
-            {
+            if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
                 imap.empty_folder(context, &mvbox_folder).await;
             }
         }
@@ -627,6 +631,11 @@ impl Job {
     }
 
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
@@ -798,7 +807,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
 
     if rendered_msg.is_encrypted && !needs_encryption {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
-        msg.save_param_to_disk(context).await;
+        msg.update_param(context).await;
     }
 
     ensure!(!recipients.is_empty(), "no recipients for smtp job set");
@@ -815,31 +824,11 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     Ok(Some(job))
 }
 
-#[derive(Debug)]
-pub enum Connection<'a> {
+pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
-    if let Some(delete_server_after) = context.get_config_delete_server_after().await {
-        let threshold_timestamp = time() - delete_server_after;
-
-        context
-            .sql
-            .query_row_optional(
-                "SELECT id FROM msgs \
-             WHERE timestamp < ? \
-             AND server_uid != 0",
-                paramsv![threshold_timestamp],
-                |row| row.get::<_, MsgId>(0),
-            )
-            .await
-    } else {
-        Ok(None)
-    }
-}
-
 async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         Some(Job::new(
@@ -1028,13 +1017,19 @@ pub async fn add(context: &Context, job: Job) {
             | Action::DeleteMsgOnImap
             | Action::MarkseenMsgOnImap
             | Action::MoveMsg => {
-                context.interrupt_inbox().await;
+                info!(context, "interrupt: imap");
+                context
+                    .interrupt_inbox(InterruptInfo::new(false, None))
+                    .await;
             }
             Action::MaybeSendLocations
             | Action::MaybeSendLocationsEnded
             | Action::SendMdn
             | Action::SendMsgToSmtp => {
-                context.interrupt_smtp().await;
+                info!(context, "interrupt: smtp");
+                context
+                    .interrupt_smtp(InterruptInfo::new(false, None))
+                    .await;
             }
         }
     }
@@ -1049,38 +1044,49 @@ pub async fn add(context: &Context, job: Job) {
 pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
-    probe_network: bool,
+    info: &InterruptInfo,
 ) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
-    let query = if !probe_network {
+
+    let query;
+    let params;
+    let t = time();
+    let m;
+    let thread_i = thread as i64;
+
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
+SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+FROM jobs
+WHERE thread=? AND foreign_id=?
+ORDER BY action DESC, added_timestamp
+LIMIT 1;
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
         // processing for first-try and after backoff-timeouts:
         // process jobs in the order they were added.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND desired_timestamp<=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#
+"#;
+        params = paramsv![thread_i, t];
     } else {
         // processing after call to dc_maybe_network():
         // process _all_ pending jobs that failed before
         // in the order of their backoff-times.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND tries>0
 ORDER BY desired_timestamp, action DESC
 LIMIT 1;
-"#
-    };
-
-    let thread_i = thread as i64;
-    let t = time();
-    let params = if !probe_network {
-        paramsv![thread_i, t]
-    } else {
-        paramsv![thread_i]
+"#;
+        params = paramsv![thread_i];
     };
 
     let job = loop {
@@ -1088,13 +1094,13 @@ LIMIT 1;
             .sql
             .query_row_optional(query, params.clone(), |row| {
                 let job = Job {
-                    job_id: row.get(0)?,
-                    action: row.get(1)?,
-                    foreign_id: row.get(2)?,
-                    desired_timestamp: row.get(5)?,
-                    added_timestamp: row.get(4)?,
-                    tries: row.get(6)?,
-                    param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
+                    job_id: row.get("id")?,
+                    action: row.get("action")?,
+                    foreign_id: row.get("foreign_id")?,
+                    desired_timestamp: row.get("desired_timestamp")?,
+                    added_timestamp: row.get("added_timestamp")?,
+                    tries: row.get("tries")?,
+                    param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
                     pending_error: None,
                 };
 
@@ -1104,8 +1110,9 @@ LIMIT 1;
 
         match job_res {
             Ok(job) => break job,
-            Err(_) => {
+            Err(err) => {
                 // Remove invalid job from the DB
+                info!(context, "cleaning up job, because of {}", err);
 
                 // TODO: improve by only doing a single query
                 match context
@@ -1116,7 +1123,7 @@ LIMIT 1;
                     Ok(id) => {
                         context
                             .sql
-                            .execute("DELETE FROM jobs WHERE id=?", paramsv![id])
+                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
                             .await
                             .ok();
                     }
@@ -1129,21 +1136,26 @@ LIMIT 1;
         }
     };
 
-    if thread == Thread::Imap {
-        if let Some(job) = job {
-            if job.action < Action::DeleteMsgOnImap {
-                load_imap_deletion_job(context)
-                    .await
-                    .unwrap_or_default()
-                    .or(Some(job))
-            } else {
-                Some(job)
-            }
-        } else {
-            load_imap_deletion_job(context).await.unwrap_or_default()
+    match thread {
+        Thread::Unknown => {
+            error!(context, "unknown thread for job");
+            None
         }
-    } else {
-        job
+        Thread::Imap => {
+            if let Some(job) = job {
+                if job.action < Action::DeleteMsgOnImap {
+                    load_imap_deletion_job(context)
+                        .await
+                        .unwrap_or_default()
+                        .or(Some(job))
+                } else {
+                    Some(job)
+                }
+            } else {
+                load_imap_deletion_job(context).await.unwrap_or_default()
+            }
+        }
+        Thread::Smtp => job,
     }
 }
 
@@ -1175,17 +1187,42 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_load_next_job() {
+    async fn test_load_next_job_two() {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_none());
 
         insert_job(&t.ctx, 1).await;
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
+        assert!(jobs.is_some());
+    }
+
+    #[async_std::test]
+    async fn test_load_next_job_one() {
+        let t = TestContext::new().await;
+
+        insert_job(&t.ctx, 1).await;
+
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_some());
     }
 }

src/key.rs

@@ -1,19 +1,20 @@
 //! Cryptographic key module
 
 use std::collections::BTreeMap;
+use std::fmt;
 use std::io::Cursor;
 
-use async_std::path::Path;
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
+use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 use crate::sql;
 
 // Re-export key types
@@ -38,6 +39,8 @@ pub enum Error {
     NoConfiguredAddr,
     #[error("Configured address is invalid: {}", _0)]
     InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[error("no data provided")]
+    Empty,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -48,8 +51,8 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
 #[async_trait]
-pub trait DcKey: Serialize + Deserializable {
-    type KeyType: Serialize + Deserializable;
+pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
+    type KeyType: Serialize + Deserializable + KeyTrait + Clone;
 
     /// Create a key from some bytes.
     fn from_slice(bytes: &[u8]) -> Result<Self::KeyType> {
@@ -66,18 +69,45 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
+    /// Create a key from an ASCII-armored string.
+    ///
+    /// Returns the key and a map of any headers which might have been set in
+    /// the ASCII-armored representation.
+    fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
+        let bytes = data.as_bytes();
+        Self::KeyType::from_armor_single(Cursor::new(bytes)).map_err(Error::Pgp)
+    }
+
     /// Load the users' default key from the database.
     async fn load_self(context: &Context) -> Result<Self::KeyType>;
 
-    /// Serialise the key to a base64 string.
-    fn to_base64(&self) -> String {
+    /// Serialise the key as bytes.
+    fn to_bytes(&self) -> Vec<u8> {
         // Not using Serialize::to_bytes() to make clear *why* it is
         // safe to ignore this error.
         // Because we write to a Vec<u8> the io::Write impls never
         // fail and we can hide this error.
         let mut buf = Vec::new();
         self.to_writer(&mut buf).unwrap();
-        base64::encode(&buf)
+        buf
+    }
+
+    /// Serialise the key to a base64 string.
+    fn to_base64(&self) -> String {
+        base64::encode(&DcKey::to_bytes(self))
+    }
+
+    /// Serialise the key to ASCII-armored representation.
+    ///
+    /// Each header line must be terminated by `\r\n`.  Only allows setting one
+    /// header as a simplification since that's the only way it's used so far.
+    // Since .to_armored_string() are actual methods on SignedPublicKey and
+    // SignedSecretKey we can not generically implement this.
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String;
+
+    /// The fingerprint for the key.
+    fn fingerprint(&self) -> Fingerprint {
+        Fingerprint::new(KeyTrait::fingerprint(self)).expect("Invalid fingerprint from rpgp")
     }
 }
 
@@ -108,6 +138,22 @@ impl DcKey for SignedPublicKey {
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to ignore this error.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
 }
 
 #[async_trait]
@@ -137,6 +183,39 @@ impl DcKey for SignedSecretKey {
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to do these unwraps.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.  The string is always ASCII.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
+}
+
+/// Deltachat extension trait for secret keys.
+///
+/// Provides some convenience wrappers only applicable to [SignedSecretKey].
+pub trait DcSecretKey {
+    /// Create a public key from a private one.
+    fn split_public_key(&self) -> Result<SignedPublicKey>;
+}
+
+impl DcSecretKey for SignedSecretKey {
+    fn split_public_key(&self) -> Result<SignedPublicKey> {
+        self.verify()?;
+        let unsigned_pubkey = SecretKeyTrait::public_key(self);
+        let signed_pubkey = unsigned_pubkey.sign(self, || "".into())?;
+        Ok(signed_pubkey)
+    }
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
@@ -172,7 +251,9 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
-            let keypair = crate::pgp::create_keypair(addr, keytype)?;
+            let keypair =
+                async_std::task::spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
+                    .await?;
             store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
             info!(
                 context,
@@ -185,198 +266,6 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     }
 }
 
-/// Cryptographic key
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum Key {
-    Public(SignedPublicKey),
-    Secret(SignedSecretKey),
-}
-
-impl From<SignedPublicKey> for Key {
-    fn from(key: SignedPublicKey) -> Self {
-        Key::Public(key)
-    }
-}
-
-impl From<SignedSecretKey> for Key {
-    fn from(key: SignedSecretKey) -> Self {
-        Key::Secret(key)
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl Key {
-    pub fn is_public(&self) -> bool {
-        match self {
-            Key::Public(_) => true,
-            Key::Secret(_) => false,
-        }
-    }
-
-    pub fn is_secret(&self) -> bool {
-        !self.is_public()
-    }
-
-    pub fn from_slice(bytes: &[u8], key_type: KeyType) -> Option<Self> {
-        if bytes.is_empty() {
-            return None;
-        }
-        let res: std::result::Result<Key, _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-            KeyType::Private => SignedSecretKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-        };
-
-        match res {
-            Ok(key) => Some(key),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn from_armored_string(
-        data: &str,
-        key_type: KeyType,
-    ) -> Option<(Self, BTreeMap<String, String>)> {
-        let bytes = data.as_bytes();
-        let res: std::result::Result<(Key, _), _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-            KeyType::Private => SignedSecretKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-        };
-
-        match res {
-            Ok(res) => Some(res),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn to_bytes(&self) -> Vec<u8> {
-        match self {
-            Key::Public(k) => k.to_bytes().unwrap_or_default(),
-            Key::Secret(k) => k.to_bytes().unwrap_or_default(),
-        }
-    }
-
-    pub fn verify(&self) -> bool {
-        match self {
-            Key::Public(k) => k.verify().is_ok(),
-            Key::Secret(k) => k.verify().is_ok(),
-        }
-    }
-
-    pub fn to_base64(&self) -> String {
-        let buf = self.to_bytes();
-        base64::encode(&buf)
-    }
-
-    pub fn to_armored_string(
-        &self,
-        headers: Option<&BTreeMap<String, String>>,
-    ) -> pgp::errors::Result<String> {
-        match self {
-            Key::Public(k) => k.to_armored_string(headers),
-            Key::Secret(k) => k.to_armored_string(headers),
-        }
-    }
-
-    /// Each header line must be terminated by `\r\n`
-    pub fn to_asc(&self, header: Option<(&str, &str)>) -> String {
-        let headers = header.map(|(key, value)| {
-            let mut m = BTreeMap::new();
-            m.insert(key.to_string(), value.to_string());
-            m
-        });
-
-        self.to_armored_string(headers.as_ref())
-            .expect("failed to serialize key")
-    }
-
-    pub async fn write_asc_to_file(
-        &self,
-        file: impl AsRef<Path>,
-        context: &Context,
-    ) -> std::io::Result<()> {
-        let file_content = self.to_asc(None).into_bytes();
-
-        let res = dc_write_file(context, &file, &file_content).await;
-        if res.is_err() {
-            error!(context, "Cannot write key to {}", file.as_ref().display());
-        }
-        res
-    }
-
-    pub fn fingerprint(&self) -> String {
-        match self {
-            Key::Public(k) => hex::encode_upper(k.fingerprint()),
-            Key::Secret(k) => hex::encode_upper(k.fingerprint()),
-        }
-    }
-
-    pub fn formatted_fingerprint(&self) -> String {
-        let rawhex = self.fingerprint();
-        dc_format_fingerprint(&rawhex)
-    }
-
-    pub fn split_key(&self) -> Option<Key> {
-        match self {
-            Key::Public(_) => None,
-            Key::Secret(k) => {
-                let pub_key = k.public_key();
-                pub_key.sign(k, || "".into()).map(Key::Public).ok()
-            }
-        }
-    }
-}
-
 /// Use of a [KeyPair] for encryption or decryption.
 ///
 /// This is used by [store_self_keypair] to know what kind of key is
@@ -426,14 +315,8 @@ pub async fn store_self_keypair(
 ) -> std::result::Result<(), SaveKeyError> {
     // Everything should really be one transaction, more refactoring
     // is needed for that.
-    let public_key = keypair
-        .public
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise public key", err))?;
-    let secret_key = keypair
-        .secret
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise secret key", err))?;
+    let public_key = DcKey::to_bytes(&keypair.public);
+    let secret_key = DcKey::to_bytes(&keypair.secret);
     context
         .sql
         .execute(
@@ -471,37 +354,73 @@ pub async fn store_self_keypair(
     Ok(())
 }
 
-/// Make a fingerprint human-readable, in hex format.
-pub fn dc_format_fingerprint(fingerprint: &str) -> String {
-    // split key into chunks of 4 with space, and 20 newline
-    let mut res = String::new();
+/// A key fingerprint
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct Fingerprint(Vec<u8>);
 
-    for (i, c) in fingerprint.chars().enumerate() {
-        if i > 0 && i % 20 == 0 {
-            res += "\n";
-        } else if i > 0 && i % 4 == 0 {
-            res += " ";
+impl Fingerprint {
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+        match v.len() {
+            20 => Ok(Fingerprint(v)),
+            _ => Err(FingerprintError::WrongLength),
         }
-
-        res += &c.to_string();
     }
 
-    res
+    /// Make a hex string from the fingerprint.
+    ///
+    /// Use [std::fmt::Display] or [ToString::to_string] to get a
+    /// human-readable formatted string.
+    pub fn hex(&self) -> String {
+        hex::encode_upper(&self.0)
+    }
 }
 
-/// Bring a human-readable or otherwise formatted fingerprint back to the 40-characters-uppercase-hex format.
-pub fn dc_normalize_fingerprint(fp: &str) -> String {
-    fp.to_uppercase()
-        .chars()
-        .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
-        .collect()
+/// Make a human-readable fingerprint.
+impl fmt::Display for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Split key into chunks of 4 with space and newline at 20 chars
+        for (i, c) in self.hex().chars().enumerate() {
+            if i > 0 && i % 20 == 0 {
+                writeln!(f)?;
+            } else if i > 0 && i % 4 == 0 {
+                write!(f, " ")?;
+            }
+            write!(f, "{}", c)?;
+        }
+        Ok(())
+    }
+}
+
+/// Parse a human-readable or otherwise formatted fingerprint.
+impl std::str::FromStr for Fingerprint {
+    type Err = FingerprintError;
+
+    fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
+        let hex_repr: String = input
+            .to_uppercase()
+            .chars()
+            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .collect();
+        let v: Vec<u8> = hex::decode(hex_repr)?;
+        let fp = Fingerprint::new(v)?;
+        Ok(fp)
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use std::convert::TryFrom;
+
+    use std::error::Error;
 
     use async_std::sync::Arc;
     use lazy_static::lazy_static;
@@ -510,16 +429,9 @@ mod tests {
         static ref KEYPAIR: KeyPair = alice_keypair();
     }
 
-    #[test]
-    fn test_normalize_fingerprint() {
-        let fingerprint = dc_normalize_fingerprint(" 1234  567890 \n AbcD abcdef ABCDEF ");
-
-        assert_eq!(fingerprint, "1234567890ABCDABCDEFABCDEF");
-    }
-
     #[test]
     fn test_from_armored_string() {
-        let (private_key, _) = Key::from_armored_string(
+        let (private_key, _) = SignedSecretKey::from_asc(
             "-----BEGIN PGP PRIVATE KEY BLOCK-----
 
 xcLYBF0fgz4BCADnRUV52V4xhSsU56ZaAn3+3oG86MZhXy4X8w14WZZDf0VJGeTh
@@ -577,63 +489,69 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 7yPJeQ==
 =KZk/
 -----END PGP PRIVATE KEY BLOCK-----",
-            KeyType::Private,
         )
-        .expect("failed to decode"); // NOTE: if you take out the ===GU1/ part, everything passes!
-        let binary = private_key.to_bytes();
-        Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        .expect("failed to decode");
+        let binary = DcKey::to_bytes(&private_key);
+        SignedSecretKey::from_slice(&binary).expect("invalid private key");
     }
 
     #[test]
-    fn test_format_fingerprint() {
-        let fingerprint = dc_format_fingerprint("1234567890ABCDABCDEFABCDEF1234567890ABCD");
+    fn test_asc_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedPublicKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
 
-        assert_eq!(
-            fingerprint,
-            "1234 5678 90AB CDAB CDEF\nABCD EF12 3456 7890 ABCD"
-        );
+        let key = KEYPAIR.secret.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedSecretKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
     }
 
     #[test]
     fn test_from_slice_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+        let public_key = KEYPAIR.public.clone();
+        let private_key = KEYPAIR.secret.clone();
 
-        let binary = public_key.to_bytes();
-        let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
+        let binary = DcKey::to_bytes(&public_key);
+        let public_key2 = SignedPublicKey::from_slice(&binary).expect("invalid public key");
         assert_eq!(public_key, public_key2);
 
-        let binary = private_key.to_bytes();
-        let private_key2 = Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        let binary = DcKey::to_bytes(&private_key);
+        let private_key2 = SignedSecretKey::from_slice(&binary).expect("invalid private key");
         assert_eq!(private_key, private_key2);
     }
 
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-
         for i in 0..4096 {
             bad_data[i] = (i & 0xff) as u8;
         }
-
         for j in 0..(4096 / 40) {
-            let bad_key = Key::from_slice(
-                &bad_data[j..j + 4096 / 2 + j],
-                if 0 != j & 1 {
-                    KeyType::Public
-                } else {
-                    KeyType::Private
-                },
-            );
-            assert!(bad_key.is_none());
+            let slice = &bad_data[j..j + 4096 / 2 + j];
+            assert!(SignedPublicKey::from_slice(slice).is_err());
+            assert!(SignedSecretKey::from_slice(slice).is_err());
         }
     }
 
+    #[test]
+    fn test_base64_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let base64 = key.to_base64();
+        let key2 = SignedPublicKey::from_base64(&base64).unwrap();
+        assert_eq!(key, key2);
+    }
+
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = dummy_context().await;
-        configure_alice_keypair(&t.ctx).await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
         assert_eq!(alice.public, pubkey);
         let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
@@ -641,9 +559,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_public() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -653,9 +570,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_secret() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -665,11 +581,10 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -686,36 +601,17 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(res0.unwrap(), res1.unwrap());
     }
 
-    #[test]
-    fn test_ascii_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
-
-        let s = public_key.to_armored_string(None).unwrap();
-        let (public_key2, _) =
-            Key::from_armored_string(&s, KeyType::Public).expect("invalid public key");
-        assert_eq!(public_key, public_key2);
-
-        let s = private_key.to_armored_string(None).unwrap();
-        println!("{}", &s);
-        let (private_key2, _) =
-            Key::from_armored_string(&s, KeyType::Private).expect("invalid private key");
-        assert_eq!(private_key, private_key2);
-    }
-
     #[test]
     fn test_split_key() {
-        let private_key = Key::from(KEYPAIR.secret.clone());
-        let public_wrapped = private_key.split_key().unwrap();
-        let public = SignedPublicKey::try_from(public_wrapped).unwrap();
-        assert_eq!(public.primary_key, KEYPAIR.public.primary_key);
+        let pubkey = KEYPAIR.secret.split_public_key().unwrap();
+        assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
     #[async_std::test]
     async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let ctx = Arc::new(t.ctx);
 
         let ctx1 = ctx.clone();
@@ -756,4 +652,49 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     //     )
     //     .unwrap();
     // }
+
+    #[test]
+    fn test_fingerprint_from_str() {
+        let res = Fingerprint::new(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+
+        let fp: Fingerprint = "0102030405060708090A0B0c0d0e0F1011121314".parse().unwrap();
+        assert_eq!(fp, res);
+
+        let fp: Fingerprint = "zzzz 0102 0304 0506\n0708090a0b0c0D0E0F1011121314 yyy"
+            .parse()
+            .unwrap();
+        assert_eq!(fp, res);
+
+        let err = "1".parse::<Fingerprint>().err().unwrap();
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
+    }
+
+    #[test]
+    fn test_fingerprint_hex() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(fp.hex(), "0102040810204080FF0A0B0C0D0E0F1011121314");
+    }
+
+    #[test]
+    fn test_fingerprint_to_string() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(
+            fp.to_string(),
+            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF 1314"
+        );
+    }
 }

src/keyring.rs

@@ -1,46 +1,92 @@
-use std::borrow::Cow;
+//! Keyring to perform rpgp operations with.
+
+use anyhow::Result;
 
-use crate::constants::KeyType;
 use crate::context::Context;
-use crate::key::Key;
-use crate::sql::Sql;
+use crate::key::{self, DcKey};
 
-#[derive(Default, Clone, Debug)]
-pub struct Keyring<'a> {
-    keys: Vec<Cow<'a, Key>>,
+/// An in-memory keyring.
+///
+/// Instances are usually constructed just for the rpgp operation and
+/// short-lived.
+#[derive(Clone, Debug, Default)]
+pub struct Keyring<T>
+where
+    T: DcKey,
+{
+    keys: Vec<T>,
 }
 
-impl<'a> Keyring<'a> {
-    pub fn add_owned(&mut self, key: Key) {
-        self.add(Cow::Owned(key))
+impl<T> Keyring<T>
+where
+    T: DcKey<KeyType = T>,
+{
+    /// New empty keyring.
+    pub fn new() -> Keyring<T> {
+        Keyring { keys: Vec::new() }
     }
 
-    pub fn add_ref(&mut self, key: &'a Key) {
-        self.add(Cow::Borrowed(key))
+    /// Create a new keyring with the the user's secret key loaded.
+    pub async fn new_self(context: &Context) -> Result<Keyring<T>, key::Error> {
+        let mut keyring: Keyring<T> = Keyring::new();
+        keyring.load_self(context).await?;
+        Ok(keyring)
     }
 
-    fn add(&mut self, key: Cow<'a, Key>) {
+    /// Load the user's key into the keyring.
+    pub async fn load_self(&mut self, context: &Context) -> Result<(), key::Error> {
+        self.add(T::load_self(context).await?);
+        Ok(())
+    }
+
+    /// Add a key to the keyring.
+    pub fn add(&mut self, key: T) {
         self.keys.push(key);
     }
 
-    pub fn keys(&self) -> &[Cow<'a, Key>] {
-        &self.keys
+    pub fn len(&self) -> usize {
+        self.keys.len()
     }
 
-    pub async fn load_self_private_for_decrypting(
-        &mut self,
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> bool {
-        sql.query_get_value(
-            context,
-            "SELECT private_key FROM keypairs ORDER BY addr=? DESC, is_default DESC;",
-            paramsv![self_addr.as_ref().to_string()],
-        )
-        .await
-        .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Private))
-        .map(|key| self.add_owned(key))
-        .is_some()
+    pub fn is_empty(&self) -> bool {
+        self.keys.is_empty()
+    }
+
+    /// A vector with reference to all the keys in the keyring.
+    pub fn keys(&self) -> &[T] {
+        &self.keys
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::key::{SignedPublicKey, SignedSecretKey};
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_keyring_add_keys() {
+        let alice = alice_keypair();
+        let mut pub_ring: Keyring<SignedPublicKey> = Keyring::new();
+        pub_ring.add(alice.public.clone());
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let mut sec_ring: Keyring<SignedSecretKey> = Keyring::new();
+        sec_ring.add(alice.secret.clone());
+        assert_eq!(sec_ring.keys(), [alice.secret]);
+    }
+
+    #[async_std::test]
+    async fn test_keyring_load_self() {
+        // new_self() implies load_self()
+        let t = TestContext::new().await;
+        t.configure_alice().await;
+        let alice = alice_keypair();
+
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -11,8 +11,6 @@ extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 
@@ -45,6 +43,7 @@ pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
+pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;

src/location.rs

@@ -530,7 +530,7 @@ pub async fn save(
             accuracy,
             ..
         } = location;
-        context
+        let (loc_id, ts) = context
             .sql
             .with_conn(move |mut conn| {
                 let mut stmt_test = conn
@@ -569,9 +569,11 @@ pub async fn save(
                         )?;
                     }
                 }
-                Ok(())
+                Ok((newest_location_id, newest_timestamp))
             })
             .await?;
+        newest_timestamp = ts;
+        newest_location_id = loc_id;
     }
 
     Ok(newest_location_id)
@@ -722,11 +724,11 @@ pub(crate) async fn job_maybe_send_locations_ended(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_kml_parse() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
 
         let xml =
             b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";

src/login_param.rs

@@ -130,10 +130,6 @@ impl LoginParam {
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
     pub async fn save_to_database(
         &self,
@@ -277,21 +273,15 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-    match certificate_checks {
-        CertificateChecks::Automatic => {
-            // Same as AcceptInvalidCertificates for now.
-            // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
-        }
-        CertificateChecks::Strict => tls_builder,
-        CertificateChecks::AcceptInvalidCertificates
-        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
+
+    if strict_tls {
+        tls_builder
+    } else {
+        tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
+            .danger_accept_invalid_certs(true)
     }
 }
 

src/lot.rs

@@ -1,5 +1,7 @@
 use deltachat_derive::{FromSql, ToSql};
 
+use crate::key::Fingerprint;
+
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
 /// Lot objects are created
@@ -14,7 +16,7 @@ pub struct Lot {
     pub(crate) timestamp: i64,
     pub(crate) state: LotState,
     pub(crate) id: u32,
-    pub(crate) fingerprint: Option<String>,
+    pub(crate) fingerprint: Option<Fingerprint>,
     pub(crate) invitenumber: Option<String>,
     pub(crate) auth: Option<String>,
 }

src/message.rs

@@ -14,7 +14,7 @@ use crate::error::{ensure, Error};
 use crate::events::Event;
 use crate::job::{self, Action};
 use crate::lot::{Lot, LotState, Meaning};
-use crate::mimeparser::SystemMessage;
+use crate::mimeparser::{FailureReport, SystemMessage};
 use crate::param::*;
 use crate::pgp::*;
 use crate::stock::StockMessage;
@@ -68,20 +68,6 @@ impl MsgId {
         self.0 == 0
     }
 
-    /// Whether the message ID is the special marker1 marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_marker1(self) -> bool {
-        self.0 == DC_MSG_ID_MARKER1
-    }
-
-    /// Whether the message ID is the special day marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_daymarker(self) -> bool {
-        self.0 == DC_MSG_ID_DAYMARKER
-    }
-
     /// Put message into trash chat and delete message text.
     ///
     /// It means the message is deleted locally, but not on the server
@@ -143,16 +129,7 @@ impl MsgId {
 
 impl std::fmt::Display for MsgId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // Would be nice if we could use match here, but no computed values in ranges.
-        if self.0 == DC_MSG_ID_MARKER1 {
-            write!(f, "Msg#Marker1")
-        } else if self.0 == DC_MSG_ID_DAYMARKER {
-            write!(f, "Msg#DayMarker")
-        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
-            write!(f, "Msg#UnknownSpecial")
-        } else {
-            write!(f, "Msg#{}", self.0)
-        }
+        write!(f, "Msg#{}", self.0)
     }
 }
 
@@ -246,6 +223,8 @@ pub struct Message {
     pub(crate) timestamp_sort: i64,
     pub(crate) timestamp_sent: i64,
     pub(crate) timestamp_rcvd: i64,
+    pub(crate) ephemeral_timer: i64,
+    pub(crate) ephemeral_timestamp: i64,
     pub(crate) text: Option<String>,
     pub(crate) rfc724_mid: String,
     pub(crate) in_reply_to: Option<String>,
@@ -255,6 +234,7 @@ pub struct Message {
     pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
+    pub(crate) error: String,
     pub(crate) param: Params,
 }
 
@@ -287,8 +267,11 @@ impl Message {
                     "    m.timestamp AS timestamp,",
                     "    m.timestamp_sent AS timestamp_sent,",
                     "    m.timestamp_rcvd AS timestamp_rcvd,",
+                    "    m.ephemeral_timer AS ephemeral_timer,",
+                    "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
+                    "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.txt AS txt,",
                     "    m.param AS param,",
@@ -314,8 +297,11 @@ impl Message {
                     msg.timestamp_sort = row.get("timestamp")?;
                     msg.timestamp_sent = row.get("timestamp_sent")?;
                     msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                    msg.ephemeral_timer = row.get("ephemeral_timer")?;
+                    msg.ephemeral_timestamp = row.get("ephemeral_timestamp")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
+                    msg.error = row.get("error")?;
                     msg.is_dc_message = row.get("msgrmsg")?;
 
                     let text;
@@ -390,7 +376,7 @@ impl Message {
                     }
 
                     if !self.id.is_unset() {
-                        self.save_param_to_disk(context).await;
+                        self.update_param(context).await;
                     }
                 }
             }
@@ -643,10 +629,10 @@ impl Message {
         if duration > 0 {
             self.param.set_int(Param::Duration, duration);
         }
-        self.save_param_to_disk(context).await;
+        self.update_param(context).await;
     }
 
-    pub async fn save_param_to_disk(&mut self, context: &Context) -> bool {
+    pub async fn update_param(&mut self, context: &Context) -> bool {
         context
             .sql
             .execute(
@@ -763,9 +749,10 @@ impl From<MessageState> for LotState {
 impl MessageState {
     pub fn can_fail(self) -> bool {
         match self {
-            MessageState::OutPreparing | MessageState::OutPending | MessageState::OutDelivered => {
-                true
-            }
+            MessageState::OutPreparing
+            | MessageState::OutPending
+            | MessageState::OutDelivered
+            | MessageState::OutMdnRcvd => true, // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
             _ => false,
         }
     }
@@ -887,6 +874,17 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         ret += "\n";
     }
 
+    if msg.ephemeral_timer != 0 {
+        ret += &format!("Ephemeral timer: {}\n", msg.ephemeral_timer);
+    }
+
+    if msg.ephemeral_timestamp != 0 {
+        ret += &format!(
+            "Expires: {}\n",
+            dc_timestamp_to_str(msg.ephemeral_timestamp)
+        );
+    }
+
     if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
@@ -937,8 +935,9 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
     }
 
     ret += "\n";
-    if let Some(err) = msg.param.get(Param::Error) {
-        ret += &format!("Error: {}", err)
+
+    if !msg.error.is_empty() {
+        ret += &format!("Error: {}", msg.error);
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -1091,6 +1090,14 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
     let mut send_event = false;
 
     for (id, curr_state, curr_blocked) in msgs.into_iter() {
+        if let Err(err) = id.start_ephemeral_timer(context).await {
+            error!(
+                context,
+                "Failed to start ephemeral timer for message {}: {}", id, err
+            );
+            continue;
+        }
+
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, id, MessageState::InSeen).await;
@@ -1251,39 +1258,44 @@ pub async fn exists(context: &Context, msg_id: MsgId) -> bool {
 
 pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id).await {
+        let error = error.map(|e| e.as_ref().to_string()).unwrap_or_default();
         if msg.state.can_fail() {
             msg.state = MessageState::OutFailed;
-        }
-        if let Some(error) = error {
-            msg.param.set(Param::Error, error.as_ref());
-            warn!(context, "Message failed: {}", error.as_ref());
+            warn!(context, "{} failed: {}", msg_id, error);
+        } else {
+            warn!(
+                context,
+                "{} seems to have failed ({}), but state is {}", msg_id, error, msg.state
+            )
         }
 
-        if context
+        match context
             .sql
             .execute(
-                "UPDATE msgs SET state=?, param=? WHERE id=?;",
-                paramsv![msg.state, msg.param.to_string(), msg_id],
+                "UPDATE msgs SET state=?, error=? WHERE id=?;",
+                paramsv![msg.state, error, msg_id],
             )
             .await
-            .is_ok()
         {
-            context.emit_event(Event::MsgFailed {
+            Ok(_) => context.emit_event(Event::MsgFailed {
                 chat_id: msg.chat_id,
                 msg_id,
-            });
+            }),
+            Err(e) => {
+                warn!(context, "{:?}", e);
+            }
         }
     }
 }
 
 /// returns Some if an event should be send
-pub async fn mdn_from_ext(
+pub async fn handle_mdn(
     context: &Context,
     from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Option<(ChatId, MsgId)> {
-    if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
+    if from_id <= DC_CONTACT_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
         return None;
     }
 
@@ -1318,10 +1330,10 @@ pub async fn mdn_from_ext(
     if let Ok((msg_id, chat_id, chat_type, msg_state)) = res {
         let mut read_by_all = false;
 
-        // if already marked as MDNS_RCVD msgstate_can_fail() returns false.
-        // however, it is important, that ret_msg_id is set above as this
-        // will allow the caller eg. to move the message away
-        if msg_state.can_fail() {
+        if msg_state == MessageState::OutPreparing
+            || msg_state == MessageState::OutPending
+            || msg_state == MessageState::OutDelivered
+        {
             let mdn_already_in_table = context
                 .sql
                 .exists(
@@ -1384,6 +1396,69 @@ pub async fn mdn_from_ext(
     None
 }
 
+/// Marks a message as failed after an ndn (non-delivery-notification) arrived.
+/// Where appropriate, also adds an info message telling the user which of the recipients of a group message failed.
+pub(crate) async fn handle_ndn(
+    context: &Context,
+    failed: &FailureReport,
+    error: Option<impl AsRef<str>>,
+) {
+    if failed.rfc724_mid.is_empty() {
+        return;
+    }
+
+    let res = context
+        .sql
+        .query_row(
+            concat!(
+                "SELECT",
+                "    m.id AS msg_id,",
+                "    c.id AS chat_id,",
+                "    c.type AS type",
+                " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+                " WHERE rfc724_mid=? AND from_id=1",
+            ),
+            paramsv![failed.rfc724_mid],
+            |row| {
+                Ok((
+                    row.get::<_, MsgId>("msg_id")?,
+                    row.get::<_, ChatId>("chat_id")?,
+                    row.get::<_, Chattype>("type")?,
+                ))
+            },
+        )
+        .await;
+    if let Err(ref err) = res {
+        info!(context, "Failed to select NDN {:?}", err);
+    }
+
+    if let Ok((msg_id, chat_id, chat_type)) = res {
+        set_msg_failed(context, msg_id, error).await;
+
+        if chat_type == Chattype::Group || chat_type == Chattype::VerifiedGroup {
+            if let Some(failed_recipient) = &failed.failed_recipient {
+                let contact_id =
+                    Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
+                if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+                    // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
+                    chat::add_info_msg(
+                        context,
+                        chat_id,
+                        context
+                            .stock_string_repl_str(
+                                StockMessage::FailedSendingTo,
+                                contact.get_display_name(),
+                            )
+                            .await,
+                    )
+                    .await;
+                    context.emit_event(Event::ChatModified(chat_id));
+                }
+            }
+        }
+    }
+}
+
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
 pub async fn get_real_msg_cnt(context: &Context) -> i32 {
     match context
@@ -1572,7 +1647,7 @@ mod tests {
     async fn test_prepare_message_and_send() {
         use crate::config::Config;
 
-        let d = test::dummy_context().await;
+        let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
         let contact = Contact::create(ctx, "", "dest@example.com")
@@ -1596,7 +1671,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_summarytext_by_raw() {
-        let d = test::dummy_context().await;
+        let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
         let some_text = Some("bla bla".to_string());

src/mimefactory.rs

@@ -9,6 +9,7 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
 use crate::location;
 use crate::message::{self, Message};
@@ -351,16 +352,47 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    let raw = message::get_summarytext_by_raw(
-                        self.msg.viewtype,
-                        self.msg.text.as_ref(),
-                        &self.msg.param,
-                        32,
-                        self.context,
-                    )
-                    .await;
-                    let raw_subject = raw.lines().next().unwrap_or_default();
-                    format!("Chat: {}", raw_subject)
+                    match chat.param.get(Param::LastSubject) {
+                        Some(last_subject) => {
+                            let subject_start = if last_subject.starts_with("Chat:") {
+                                0
+                            } else {
+                                // "Antw:" is the longest abbreviation in
+                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+                                // so look at the first _5_ characters:
+                                match last_subject.chars().take(5).position(|c| c == ':') {
+                                    Some(prefix_end) => prefix_end + 1,
+                                    None => 0,
+                                }
+                            };
+                            format!(
+                                "Re: {}",
+                                last_subject
+                                    .chars()
+                                    .skip(subject_start)
+                                    .collect::<String>()
+                                    .trim()
+                            )
+                        }
+                        None => {
+                            let self_name = match self.context.get_config(Config::Displayname).await
+                            {
+                                Some(name) => name,
+                                None => self
+                                    .context
+                                    .get_config(Config::Addr)
+                                    .await
+                                    .unwrap_or_default(),
+                            };
+
+                            self.context
+                                .stock_string_repl_str(
+                                    StockMessage::SubjectForNewContact,
+                                    self_name,
+                                )
+                                .await
+                        }
+                    }
                 }
             }
             Loaded::MDN { .. } => self
@@ -409,6 +441,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
+        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -463,7 +497,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let force_plaintext = self.should_force_plaintext();
         let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
-        let mut encrypt_helper = EncryptHelper::new(self.context).await?;
+        let encrypt_helper = EncryptHelper::new(self.context).await?;
 
         let subject = encode_words(&subject_str);
 
@@ -493,6 +527,14 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
         };
 
+        let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(self.context).await?;
+        if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
+            protected_headers.push(Header::new(
+                "Ephemeral-Timer".to_string(),
+                duration.to_string(),
+            ));
+        }
+
         // we could also store the message-id in the protected headers
         // which would probably help to survive providers like
         // Outlook.com or hotmail which mangle the Message-ID.
@@ -560,7 +602,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
 
             let encrypted = encrypt_helper
-                .encrypt(self.context, min_verified, message, &peerstates)
+                .encrypt(self.context, min_verified, message, peerstates)
                 .await?;
 
             outer_message = outer_message
@@ -743,6 +785,12 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     "location-streaming-enabled".into(),
                 ));
             }
+            SystemMessage::EphemeralTimerChanged => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "ephemeral-timer-changed".to_string(),
+                ));
+            }
             SystemMessage::AutocryptSetupMessage => {
                 unprotected_headers
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
@@ -1174,6 +1222,10 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chatlist::Chatlist;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::mimeparser::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1181,6 +1233,9 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
+        assert!(!display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1192,6 +1247,25 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
+    #[test]
+    fn test_render_email_address_noescape() {
+        let display_name = "a space";
+        let addr = "x@y.org";
+
+        assert!(display_name.is_ascii());
+        assert!(display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+        let s = format!(
+            "{}",
+            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
+        );
+
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        assert_eq!(s, "a space <x@y.org>");
+    }
+
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1234,4 +1308,193 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
+
+    #[async_std::test]
+    async fn test_subject() {
+        // 1.: Receive a mail from an MUA or Delta Chat
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Infos: 42"
+        );
+
+        // 2. Receive a message from Delta Chat when we did not send any messages before
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        // 3. Send the first message to a new contact
+        let t = TestContext::new_alice().await;
+
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::Displayname, Some("Alice"))
+            .await
+            .unwrap();
+        assert_eq!(first_subject_str(t).await, "Message from Alice");
+
+        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+    }
+
+    async fn first_subject_str(t: TestContext) -> String {
+        let contact_id =
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.com", Origin::ManuallyCreated)
+                .await
+                .unwrap()
+                .0;
+
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+
+        mf.subject_str().await
+    }
+
+    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+        let t = TestContext::new_alice().await;
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        mf.subject_str().await
+    }
+
+    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+        context
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(context, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        new_msg
+    }
+
+    #[async_std::test]
+    // This test could still be extended
+    async fn test_render_reply() {
+        let t = TestContext::new_alice().await;
+        let context = &t.ctx;
+
+        let msg = incoming_msg_to_reply_msg(
+            b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            context,
+        )
+        .await;
+
+        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+
+        let recipients = mimefactory.recipients();
+        assert_eq!(recipients, vec!["charlie@example.com"]);
+
+        let rendered_msg = mimefactory.render().await.unwrap();
+
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        assert_eq!(
+            mail.headers
+                .iter()
+                .find(|h| h.get_key() == "MIME-Version")
+                .unwrap()
+                .get_value(),
+            "1.0"
+        );
+
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+            .await
+            .unwrap();
+    }
 }

src/mimeparser.rs

@@ -3,6 +3,7 @@ use std::future::Future;
 use std::pin::Pin;
 
 use deltachat_derive::{FromSql, ToSql};
+use lazy_static::lazy_static;
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 
@@ -17,6 +18,7 @@ use crate::e2ee;
 use crate::error::{bail, Result};
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
 use crate::param::*;
@@ -44,7 +46,7 @@ pub struct MimeMessage {
     pub from: Vec<SingleInfo>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
-    pub signatures: HashSet<String>,
+    pub signatures: HashSet<Fingerprint>,
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
@@ -52,7 +54,8 @@ pub struct MimeMessage {
     pub message_kml: Option<location::Kml>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
-    pub(crate) reports: Vec<Report>,
+    pub(crate) mdn_reports: Vec<Report>,
+    pub(crate) failure_report: Option<FailureReport>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -73,6 +76,9 @@ pub enum SystemMessage {
     SecurejoinMessage = 7,
     LocationStreamingEnabled = 8,
     LocationOnly = 9,
+
+    /// Chat ephemeral message timer is changed.
+    EphemeralTimerChanged = 10,
 }
 
 impl Default for SystemMessage {
@@ -175,14 +181,16 @@ impl MimeMessage {
             signatures,
             gossipped_addr,
             is_forwarded: false,
-            reports: Vec::new(),
+            mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
             user_avatar: None,
             group_avatar: None,
+            failure_report: None,
         };
         parser.parse_mime_recursive(context, &mail).await?;
+        parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
         Ok(parser)
@@ -209,6 +217,8 @@ impl MimeMessage {
         } else if let Some(value) = self.get(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
+            } else if value == "ephemeral-timer-changed" {
+                self.is_system_message = SystemMessage::EphemeralTimerChanged;
             }
         }
         Ok(())
@@ -252,7 +262,8 @@ impl MimeMessage {
                 self.parts[0].msg = "".to_string();
 
                 // swap new with old
-                std::mem::replace(&mut self.parts[0], filepart);
+                self.parts.push(filepart); // push to the end
+                let _ = self.parts.swap_remove(0); // drops first element, replacing it with the last one in O(1)
             }
         }
     }
@@ -352,7 +363,7 @@ impl MimeMessage {
         // just have send a message in the subject with an empty body.
         // Besides, we want to show something in case our incoming-processing
         // failed to properly handle an incoming message.
-        if self.parts.is_empty() && self.reports.is_empty() {
+        if self.parts.is_empty() && self.mdn_reports.is_empty() {
             let mut part = Part::default();
             part.typ = Viewtype::Text;
 
@@ -526,6 +537,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
+                part.error = "Decryption failed".to_string();
 
                 self.parts.push(part);
 
@@ -548,10 +560,10 @@ impl MimeMessage {
             (mime::MULTIPART, "report") => {
                 /* RFC 6522: the first part is for humans, the second for machines */
                 if mail.subparts.len() >= 2 {
-                    if let Some(report_type) = mail.ctype.params.get("report-type") {
-                        if report_type == "disposition-notification" {
+                    match mail.ctype.params.get("report-type").map(|s| s as &str) {
+                        Some("disposition-notification") => {
                             if let Some(report) = self.process_report(context, mail)? {
-                                self.reports.push(report);
+                                self.mdn_reports.push(report);
                             }
 
                             // Add MDN part so we can track it, avoid
@@ -563,9 +575,21 @@ impl MimeMessage {
                             self.parts.push(part);
 
                             any_part_added = true;
-                        } else {
-                            /* eg. `report-type=delivery-status`;
-                            maybe we should show them as a little error icon */
+                        }
+                        // Some providers, e.g. Tiscali, forget to set the report-type. So, if it's None, assume that it might be delivery-status
+                        Some("delivery-status") | None => {
+                            if let Some(report) = self.process_delivery_status(context, mail)? {
+                                self.failure_report = Some(report);
+                            }
+
+                            // Add all parts (we need another part, preferrably text/plain, to show as an error message)
+                            for cur_data in mail.subparts.iter() {
+                                if self.parse_mime_recursive(context, cur_data).await? {
+                                    any_part_added = true;
+                                }
+                            }
+                        }
+                        Some(_) => {
                             if let Some(first) = mail.subparts.iter().next() {
                                 any_part_added = self.parse_mime_recursive(context, first).await?;
                             }
@@ -838,24 +862,118 @@ impl MimeMessage {
         Ok(None)
     }
 
-    /// Handle reports (only MDNs for now)
-    pub async fn handle_reports(&self, context: &Context, from_id: u32, sent_timestamp: i64) {
-        if self.reports.is_empty() {
-            return;
+    fn process_delivery_status(
+        &self,
+        context: &Context,
+        report: &mailparse::ParsedMail<'_>,
+    ) -> Result<Option<FailureReport>> {
+        // parse as mailheaders
+        if let Some(original_msg) = report
+            .subparts
+            .iter()
+            .find(|p| p.ctype.mimetype.contains("rfc822") || p.ctype.mimetype == "message/global")
+        {
+            let report_body = original_msg.get_body_raw()?;
+            let (report_fields, _) = mailparse::parse_headers(&report_body)?;
+
+            if let Some(original_message_id) = report_fields
+                .get_header_value(HeaderDef::MessageId)
+                .and_then(|v| parse_message_id(&v).ok())
+            {
+                let mut to_list = get_all_addresses_from_header(&report.headers, |header_key| {
+                    header_key == "x-failed-recipients"
+                });
+                let to = if to_list.len() == 1 {
+                    Some(to_list.pop().unwrap())
+                } else {
+                    None // We do not know which recipient failed
+                };
+
+                return Ok(Some(FailureReport {
+                    rfc724_mid: original_message_id,
+                    failed_recipient: to.map(|s| s.addr),
+                }));
+            }
+
+            warn!(
+                context,
+                "ignoring unknown ndn-notification, Message-Id: {:?}",
+                report_fields.get_header_value(HeaderDef::MessageId)
+            );
         }
 
-        for report in &self.reports {
+        Ok(None)
+    }
+
+    /// Some providers like GMX and Yahoo do not send standard NDNs (Non Delivery notifications).
+    /// If you improve heuristics here you might also have to change prefetch_should_download() in imap/mod.rs.
+    /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
+    async fn heuristically_parse_ndn(&mut self, context: &Context) -> Option<()> {
+        let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
+            let from = from.to_ascii_lowercase();
+            from.contains("mailer-daemon") || from.contains("mail-daemon")
+        } else {
+            false
+        };
+        if maybe_ndn && self.failure_report.is_none() {
+            lazy_static! {
+                static ref RE: regex::Regex = regex::Regex::new(r"Message-ID:(.*)").unwrap();
+            }
+            for captures in self
+                .parts
+                .iter()
+                .filter_map(|part| part.msg_raw.as_ref())
+                .flat_map(|part| part.lines())
+                .filter_map(|line| RE.captures(line))
+            {
+                if let Ok(original_message_id) = parse_message_id(&captures[1]) {
+                    if let Ok(Some(_)) =
+                        message::rfc724_mid_exists(context, &original_message_id).await
+                    {
+                        self.failure_report = Some(FailureReport {
+                            rfc724_mid: original_message_id,
+                            failed_recipient: None,
+                        })
+                    }
+                }
+            }
+        }
+        None // Always return None, we just return anything so that we can use the '?' operator.
+    }
+
+    /// Handle reports
+    /// (MDNs = Message Disposition Notification, the message was read
+    /// and NDNs = Non delivery notification, the message could not be delivered)
+    pub async fn handle_reports(
+        &self,
+        context: &Context,
+        from_id: u32,
+        sent_timestamp: i64,
+        parts: &[Part],
+    ) {
+        for report in &self.mdn_reports {
             for original_message_id in
                 std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
             {
                 if let Some((chat_id, msg_id)) =
-                    message::mdn_from_ext(context, from_id, original_message_id, sent_timestamp)
-                        .await
+                    message::handle_mdn(context, from_id, original_message_id, sent_timestamp).await
                 {
                     context.emit_event(Event::MsgRead { chat_id, msg_id });
                 }
             }
         }
+
+        if let Some(failure_report) = &self.failure_report {
+            let error = parts.iter().find(|p| p.typ == Viewtype::Text).map(|p| {
+                let msg = &p.msg;
+                match msg.find("\n--- ") {
+                    Some(footer_start) => &msg[..footer_start],
+                    None => msg,
+                }
+                .trim()
+            });
+            message::handle_ndn(context, failure_report, error).await
+        }
     }
 }
 
@@ -912,6 +1030,12 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
+#[derive(Debug)]
+pub(crate) struct FailureReport {
+    pub rfc724_mid: String,
+    pub failed_recipient: Option<String>,
+}
+
 pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
     // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
     let mut msgids = Vec::new();
@@ -955,6 +1079,7 @@ pub struct Part {
     pub bytes: usize,
     pub param: Params,
     org_filename: Option<String>,
+    pub error: String,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1117,7 +1242,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_mimeparser_crash() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1129,7 +1254,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_rfc724_mid_exists() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/mail_with_message_id.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1143,7 +1268,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_rfc724_mid_not_exists() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1201,7 +1326,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_parse_first_addr() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello@one.org, world@two.org\n\
                     Chat-Disposition-Notification-To: wrong\n\
                     Content-Type: text/plain\n\
@@ -1222,7 +1347,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_mimeparser_with_context() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello\n\
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
@@ -1272,7 +1397,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_mimeparser_with_avatars() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
@@ -1315,7 +1440,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_mimeparser_message_kml() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Chat-Version: 1.0\n\
 From: foo <foo@example.org>\n\
 To: bar <bar@example.org>\n\
@@ -1360,7 +1485,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\
 
     #[async_std::test]
     async fn test_parse_mdn() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1401,7 +1526,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
+        assert_eq!(message.mdn_reports.len(), 1);
     }
 
     /// Test parsing multiple MDNs combined in a single message.
@@ -1410,7 +1535,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
     /// multipart MIME messages.
     #[async_std::test]
     async fn test_parse_multiple_mdns() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1481,12 +1606,12 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 2);
-        assert_eq!(message.reports.len(), 2);
+        assert_eq!(message.mdn_reports.len(), 2);
     }
 
     #[async_std::test]
     async fn test_parse_mdn_with_additional_message_ids() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1528,17 +1653,20 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
-        assert_eq!(message.reports[0].original_message_id, "foo@example.org");
+        assert_eq!(message.mdn_reports.len(), 1);
         assert_eq!(
-            &message.reports[0].additional_message_ids,
+            message.mdn_reports[0].original_message_id,
+            "foo@example.org"
+        );
+        assert_eq!(
+            &message.mdn_reports[0].additional_message_ids,
             &["foo@example.com", "foo@example.net"]
         );
     }
 
     #[async_std::test]
     async fn test_parse_inline_attachment() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
 From: sender@example.com
 To: receiver@example.com
@@ -1578,7 +1706,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 
     #[async_std::test]
     async fn parse_inline_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"Message-ID: <foobar@example.org>
 From: foo <foo@example.org>
 Subject: example
@@ -1624,7 +1752,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
     #[async_std::test]
     async fn parse_thunderbird_html_embedded_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"To: Alice <alice@example.org>
 From: Bob <bob@example.org>
 Subject: Test subject
@@ -1697,7 +1825,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
     // Outlook specifies filename in the "name" attribute of Content-Type
     #[async_std::test]
     async fn parse_outlook_html_embedded_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br##"From: Anonymous <anonymous@example.org>
 To: Anonymous <anonymous@example.org>
 Subject: Delta Chat is great stuff!

src/oauth2.rs

@@ -1,12 +1,16 @@
 //! OAuth 2 module
 
+use regex::Regex;
 use std::collections::HashMap;
 
+use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::provider;
+use crate::provider::Oauth2Authorizer;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
@@ -15,6 +19,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
+    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -24,8 +29,11 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
+    mx_pattern: None,
 };
 
+const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -33,6 +41,7 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
+    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -53,7 +62,7 @@ pub async fn dc_get_oauth2_url(
     addr: impl AsRef<str>,
     redirect_uri: impl AsRef<str>,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
             .set_raw_config(
@@ -81,7 +90,7 @@ pub async fn dc_get_oauth2_access_token(
     code: impl AsRef<str>,
     regenerate: bool,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -239,7 +248,7 @@ pub async fn dc_get_oauth2_addr(
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    let oauth2 = Oauth2::from_address(addr.as_ref()).await?;
     oauth2.get_userinfo?;
 
     if let Some(access_token) =
@@ -263,23 +272,56 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    fn from_address(addr: impl AsRef<str>) -> Option<Self> {
+    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
         let addr_normalized = normalize_addr(addr.as_ref());
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            match domain {
-                "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
-                | "narod.ru" => Some(OAUTH2_YANDEX),
-                _ => None,
+            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
+                match &provider.oauth2_authorizer {
+                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
+                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
+                    None => None, // provider known to not support oauth2, no mx-lookup required
+                }
+            } else {
+                Oauth2::lookup_mx(domain).await
             }
         } else {
             None
         }
     }
 
+    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
+        if let Ok(resolver) = resolver(
+            config::ResolverConfig::default(),
+            config::ResolverOpts::default(),
+        )
+        .await
+        {
+            for provider in OAUTH2_PROVIDERS.iter() {
+                if let Some(pattern) = provider.mx_pattern {
+                    let re = Regex::new(pattern).unwrap();
+
+                    let mut fqdn: String = String::from(domain.as_ref());
+                    if !fqdn.ends_with('.') {
+                        fqdn.push_str(".");
+                    }
+
+                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
+                        for rr in res.iter() {
+                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
+                                return Some(provider.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
         let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
@@ -362,25 +404,39 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_oauth_from_address() {
-        assert_eq!(Oauth2::from_address("hello@gmail.com"), Some(OAUTH2_GMAIL));
+    #[async_std::test]
+    async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com"),
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com"),
+            Oauth2::from_address("hello@googlemail.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.com").await,
+            Some(OAUTH2_YANDEX)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address("hello@yandex.ru"), Some(OAUTH2_YANDEX));
 
-        assert_eq!(Oauth2::from_address("hello@web.de"), None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
+    }
+
+    #[async_std::test]
+    async fn test_oauth_from_mx() {
+        assert_eq!(
+            Oauth2::from_address("hello@google.com").await,
+            Some(OAUTH2_GMAIL)
+        );
     }
 
     #[async_std::test]
     async fn test_dc_get_oauth2_addr() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_addr(&ctx.ctx, addr, code).await;
@@ -390,7 +446,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_url() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
         let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
@@ -400,7 +456,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_token() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false).await;

src/param.rs

@@ -65,9 +65,6 @@ pub enum Param {
     /// For Messages
     Arg4 = b'H',
 
-    /// For Messages
-    Error = b'L',
-
     /// For Messages
     AttachGroupImage = b'A',
 
@@ -103,6 +100,9 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
+    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    LastSubject = b't',
+
     /// For Chats
     Devicetalk = b'D',
 
@@ -414,7 +414,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_params_file_fs_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
@@ -424,7 +424,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_params_file_blob() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
@@ -435,7 +435,7 @@ mod tests {
     // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
     #[async_std::test]
     async fn test_params_get_fileparam() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let fname = t.dir.path().join("foo");
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());

src/peerstate.rs

@@ -1,14 +1,12 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
 use std::collections::HashSet;
-use std::convert::TryFrom;
 use std::fmt;
 
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
-use crate::constants::*;
 use crate::context::Context;
-use crate::key::{Key, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
 
 #[derive(Debug)]
@@ -32,13 +30,13 @@ pub struct Peerstate<'a> {
     pub last_seen: i64,
     pub last_seen_autocrypt: i64,
     pub prefer_encrypt: EncryptPreference,
-    pub public_key: Option<Key>,
-    pub public_key_fingerprint: Option<String>,
-    pub gossip_key: Option<Key>,
+    pub public_key: Option<SignedPublicKey>,
+    pub public_key_fingerprint: Option<Fingerprint>,
+    pub gossip_key: Option<SignedPublicKey>,
     pub gossip_timestamp: i64,
-    pub gossip_key_fingerprint: Option<String>,
-    pub verified_key: Option<Key>,
-    pub verified_key_fingerprint: Option<String>,
+    pub gossip_key_fingerprint: Option<Fingerprint>,
+    pub verified_key: Option<SignedPublicKey>,
+    pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
     pub degrade_event: Option<DegradeEvent>,
 }
@@ -127,7 +125,7 @@ impl<'a> Peerstate<'a> {
         res.last_seen_autocrypt = message_time;
         res.to_save = Some(ToSave::All);
         res.prefer_encrypt = header.prefer_encrypt;
-        res.public_key = Some(Key::from(header.public_key.clone()));
+        res.public_key = Some(header.public_key.clone());
         res.recalc_fingerprint();
 
         res
@@ -138,7 +136,7 @@ impl<'a> Peerstate<'a> {
 
         res.gossip_timestamp = message_time;
         res.to_save = Some(ToSave::All);
-        res.gossip_key = Some(Key::from(gossip_header.public_key.clone()));
+        res.gossip_key = Some(gossip_header.public_key.clone());
         res.recalc_fingerprint();
 
         res
@@ -152,7 +150,7 @@ impl<'a> Peerstate<'a> {
     pub async fn from_fingerprint(
         context: &'a Context,
         _sql: &Sql,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
     ) -> Option<Peerstate<'a>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
@@ -161,13 +159,8 @@ impl<'a> Peerstate<'a> {
                      WHERE public_key_fingerprint=? COLLATE NOCASE \
                      OR gossip_key_fingerprint=? COLLATE NOCASE  \
                      ORDER BY public_key_fingerprint=? DESC;";
-
-        Self::from_stmt(
-            context,
-            query,
-            paramsv![fingerprint, fingerprint, fingerprint],
-        )
-        .await
+        let fp = fingerprint.hex();
+        Self::from_stmt(context, query, paramsv![fp, fp, fp]).await
     }
 
     async fn from_stmt(
@@ -190,45 +183,30 @@ impl<'a> Peerstate<'a> {
                 res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
                 res.gossip_timestamp = row.get(5)?;
 
-                res.public_key_fingerprint = row.get(7)?;
-                if res
-                    .public_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.public_key_fingerprint = None;
-                }
-                res.gossip_key_fingerprint = row.get(8)?;
-                if res
-                    .gossip_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.gossip_key_fingerprint = None;
-                }
-                res.verified_key_fingerprint = row.get(10)?;
-                if res
-                    .verified_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.verified_key_fingerprint = None;
-                }
+                res.public_key_fingerprint = row
+                    .get::<_, Option<String>>(7)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.gossip_key_fingerprint = row
+                    .get::<_, Option<String>>(8)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.verified_key_fingerprint = row
+                    .get::<_, Option<String>>(10)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
                 res.public_key = row
                     .get(4)
                     .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
+                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
                 res.gossip_key = row
                     .get(6)
                     .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
+                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
                 res.verified_key = row
                     .get(9)
                     .ok()
-                    .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Public));
+                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
 
                 Ok(res)
             })
@@ -300,8 +278,8 @@ impl<'a> Peerstate<'a> {
                 self.to_save = Some(ToSave::All)
             }
 
-            if self.public_key.as_ref() != Some(&Key::from(header.public_key.clone())) {
-                self.public_key = Some(Key::from(header.public_key.clone()));
+            if self.public_key.as_ref() != Some(&header.public_key) {
+                self.public_key = Some(header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All);
             }
@@ -316,9 +294,8 @@ impl<'a> Peerstate<'a> {
         if message_time > self.gossip_timestamp {
             self.gossip_timestamp = message_time;
             self.to_save = Some(ToSave::Timestamps);
-            let hdr_key = Key::from(gossip_header.public_key.clone());
-            if self.gossip_key.as_ref() != Some(&hdr_key) {
-                self.gossip_key = Some(hdr_key);
+            if self.gossip_key.as_ref() != Some(&gossip_header.public_key) {
+                self.gossip_key = Some(gossip_header.public_key.clone());
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All)
             }
@@ -346,11 +323,9 @@ impl<'a> Peerstate<'a> {
 
     pub fn render_gossip_header(&self, min_verified: PeerstateVerifiedStatus) -> Option<String> {
         if let Some(key) = self.peek_key(min_verified) {
-            // TODO: avoid cloning
-            let public_key = SignedPublicKey::try_from(key.clone()).ok()?;
             let header = Aheader::new(
                 self.addr.clone(),
-                public_key,
+                key.clone(), // TODO: avoid cloning
                 // Autocrypt 1.1.0 specification says that
                 // `prefer-encrypt` attribute SHOULD NOT be included,
                 // but we include it anyway to propagate encryption
@@ -367,7 +342,16 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&Key> {
+    pub fn take_key(mut self, min_verified: PeerstateVerifiedStatus) -> Option<SignedPublicKey> {
+        match min_verified {
+            PeerstateVerifiedStatus::BidirectVerified => self.verified_key.take(),
+            PeerstateVerifiedStatus::Unverified => {
+                self.public_key.take().or_else(|| self.gossip_key.take())
+            }
+        }
+    }
+
+    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&SignedPublicKey> {
         match min_verified {
             PeerstateVerifiedStatus::BidirectVerified => self.verified_key.as_ref(),
             PeerstateVerifiedStatus::Unverified => self
@@ -380,7 +364,7 @@ impl<'a> Peerstate<'a> {
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
         verified: PeerstateVerifiedStatus,
     ) -> bool {
         if verified == PeerstateVerifiedStatus::BidirectVerified {
@@ -438,10 +422,10 @@ impl<'a> Peerstate<'a> {
                     self.public_key.as_ref().map(|k| k.to_bytes()),
                     self.gossip_timestamp,
                     self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    self.public_key_fingerprint,
-                    self.gossip_key_fingerprint,
+                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    self.verified_key_fingerprint,
+                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.addr,
                 ],
             ).await?;
@@ -462,15 +446,18 @@ impl<'a> Peerstate<'a> {
         Ok(())
     }
 
-    pub fn has_verified_key(&self, fingerprints: &HashSet<String>) -> bool {
-        if self.verified_key.is_some() && self.verified_key_fingerprint.is_some() {
-            let vkc = self.verified_key_fingerprint.as_ref().unwrap();
-            if fingerprints.contains(vkc) {
-                return true;
-            }
+    pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
+        if let Some(vkc) = &self.verified_key_fingerprint {
+            fingerprints.contains(vkc) && self.verified_key.is_some()
+        } else {
+            false
         }
+    }
+}
 
-        false
+impl From<crate::key::FingerprintError> for rusqlite::Error {
+    fn from(_source: crate::key::FingerprintError) -> Self {
+        Self::InvalidColumnType(0, "Invalid fingerprint".into(), rusqlite::types::Type::Text)
     }
 }
 
@@ -483,10 +470,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
             context: &ctx.ctx,
@@ -526,9 +513,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_double_create() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let peerstate = Peerstate {
             context: &ctx.ctx,
@@ -559,10 +546,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_with_empty_gossip_key_save_to_db() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
-        let pub_key = crate::key::Key::from(alice_keypair().public);
+        let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
             context: &ctx.ctx,

src/pgp.rs

@@ -1,7 +1,6 @@
 //! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
 
 use std::collections::{BTreeMap, HashSet};
-use std::convert::TryInto;
 use std::io;
 use std::io::Cursor;
 
@@ -19,8 +18,8 @@ use rand::{thread_rng, CryptoRng, Rng};
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::key::*;
-use crate::keyring::*;
+use crate::key::{DcKey, Fingerprint};
+use crate::keyring::Keyring;
 
 pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
 pub const HEADER_SETUPCODE: &str = "passphrase-begin";
@@ -238,124 +237,142 @@ fn select_pk_for_encryption(key: &SignedPublicKey) -> Option<SignedPublicKeyOrSu
 
 /// Encrypts `plain` text using `public_keys_for_encryption`
 /// and signs it using `private_key_for_signing`.
-pub fn pk_encrypt(
+pub async fn pk_encrypt(
     plain: &[u8],
-    public_keys_for_encryption: &Keyring,
-    private_key_for_signing: Option<&Key>,
+    public_keys_for_encryption: Keyring<SignedPublicKey>,
+    private_key_for_signing: Option<SignedSecretKey>,
 ) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
-    let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            key.as_ref()
-                .try_into()
-                .ok()
-                .and_then(select_pk_for_encryption)
-        })
-        .collect();
-    let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    let mut rng = thread_rng();
+    async_std::task::spawn_blocking(move || {
+        let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
+            .keys()
+            .iter()
+            .filter_map(|key| select_pk_for_encryption(key))
+            .collect();
+        let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
-    // TODO: measure time
-    let encrypted_msg = if let Some(private_key) = private_key_for_signing {
-        let skey: &SignedSecretKey = private_key
-            .try_into()
-            .map_err(|_| format_err!("Invalid private key"))?;
+        let mut rng = thread_rng();
 
-        lit_msg
-            .sign(skey, || "".into(), Default::default())
-            .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
-            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
-    } else {
-        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
-    };
+        // TODO: measure time
+        let encrypted_msg = if let Some(ref skey) = private_key_for_signing {
+            lit_msg
+                .sign(skey, || "".into(), Default::default())
+                .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
+                .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
+        } else {
+            lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
+        };
 
-    let msg = encrypted_msg?;
-    let encoded_msg = msg.to_armored_string(None)?;
+        let msg = encrypted_msg?;
+        let encoded_msg = msg.to_armored_string(None)?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
+    .await
 }
 
+/// Decrypts the message with keys from the private key keyring.
+///
+/// Receiver private keys are provided in
+/// `private_keys_for_decryption`.
+///
+/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// of all keys from the `public_keys_for_validation` keyring that
+/// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
-pub fn pk_decrypt(
-    ctext: &[u8],
-    private_keys_for_decryption: &Keyring,
-    public_keys_for_validation: &Keyring,
-    ret_signature_fingerprints: Option<&mut HashSet<String>>,
+pub async fn pk_decrypt(
+    ctext: Vec<u8>,
+    private_keys_for_decryption: Keyring<SignedSecretKey>,
+    public_keys_for_validation: Keyring<SignedPublicKey>,
+    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
 ) -> Result<Vec<u8>> {
-    let (msg, _) = Message::from_armor_single(Cursor::new(ctext))?;
-    let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption
-        .keys()
-        .iter()
-        .filter_map(|key| {
-            let k: &Key = &key;
-            k.try_into().ok()
-        })
-        .collect();
+    let msgs = async_std::task::spawn_blocking(move || {
+        let cursor = Cursor::new(ctext);
+        let (msg, _) = Message::from_armor_single(cursor)?;
 
-    let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-    ensure!(!msgs.is_empty(), "No valid messages found");
+        let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption.keys().iter().collect();
 
-    let dec_msg = &msgs[0];
+        let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
+        decryptor.collect::<pgp::errors::Result<Vec<_>>>()
+    })
+    .await?;
 
-    if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-        if !public_keys_for_validation.keys().is_empty() {
-            let pkeys: Vec<&SignedPublicKey> = public_keys_for_validation
-                .keys()
-                .iter()
-                .filter_map(|key| {
-                    let k: &Key = &key;
-                    k.try_into().ok()
+    if let Some(msg) = msgs.into_iter().next() {
+        // get_content() will decompress the message if needed,
+        // but this avoids decompressing it again to check signatures
+        let msg = msg.decompress()?;
+
+        let content = match msg.get_content()? {
+            Some(content) => content,
+            None => bail!("The decrypted message is empty"),
+        };
+
+        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
+            if !public_keys_for_validation.is_empty() {
+                let fingerprints = async_std::task::spawn_blocking(move || {
+                    let pkeys = public_keys_for_validation.keys();
+
+                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                        for pkey in pkeys {
+                            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                                let fp = DcKey::fingerprint(pkey);
+                                fingerprints.push(fp);
+                            }
+                        }
+                    }
+                    fingerprints
                 })
-                .collect();
+                .await;
 
-            for pkey in &pkeys {
-                if dec_msg.verify(&pkey.primary_key).is_ok() {
-                    let fp = hex::encode_upper(pkey.fingerprint());
-                    ret_signature_fingerprints.insert(fp);
-                }
+                ret_signature_fingerprints.extend(fingerprints);
             }
         }
-    }
-
-    match dec_msg.get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
+        Ok(content)
+    } else {
+        bail!("No valid messages found");
     }
 }
 
 /// Symmetric encryption.
-pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
-    let mut rng = thread_rng();
+pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
+    let passphrase = passphrase.to_string();
 
-    let s2k = StringToKey::new_default(&mut rng);
-    let msg =
-        lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase.into())?;
+    async_std::task::spawn_blocking(move || {
+        let mut rng = thread_rng();
+        let s2k = StringToKey::new_default(&mut rng);
+        let msg =
+            lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase)?;
 
-    let encoded_msg = msg.to_armored_string(None)?;
+        let encoded_msg = msg.to_armored_string(None)?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
+    .await
 }
 
 /// Symmetric decryption.
-pub fn symm_decrypt<T: std::io::Read + std::io::Seek>(
+pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     ctext: T,
 ) -> Result<Vec<u8>> {
     let (enc_msg, _) = Message::from_armor_single(ctext)?;
-    let decryptor = enc_msg.decrypt_with_password(|| passphrase.into())?;
 
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-    ensure!(!msgs.is_empty(), "No valid messages found");
+    let passphrase = passphrase.to_string();
+    async_std::task::spawn_blocking(move || {
+        let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
-    match msgs[0].get_content()? {
-        Some(content) => Ok(content),
-        None => bail!("Decrypted message is empty"),
-    }
+        let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
+        ensure!(!msgs.is_empty(), "No valid messages found");
+
+        match msgs[0].get_content()? {
+            Some(content) => Ok(content),
+            None => bail!("Decrypted message is empty"),
+        }
+    })
+    .await
 }
 
 #[cfg(test)]
@@ -408,10 +425,10 @@ mod tests {
 
     /// [Key] objects to use in tests.
     struct TestKeys {
-        alice_secret: Key,
-        alice_public: Key,
-        bob_secret: Key,
-        bob_public: Key,
+        alice_secret: SignedSecretKey,
+        alice_public: SignedPublicKey,
+        bob_secret: SignedSecretKey,
+        bob_public: SignedPublicKey,
     }
 
     impl TestKeys {
@@ -419,10 +436,10 @@ mod tests {
             let alice = alice_keypair();
             let bob = bob_keypair();
             TestKeys {
-                alice_secret: Key::from(alice.secret.clone()),
-                alice_public: Key::from(alice.public.clone()),
-                bob_secret: Key::from(bob.secret.clone()),
-                bob_public: Key::from(bob.public.clone()),
+                alice_secret: alice.secret.clone(),
+                alice_public: alice.public.clone(),
+                bob_secret: bob.secret.clone(),
+                bob_public: bob.public.clone(),
             }
         }
     }
@@ -436,18 +453,18 @@ mod tests {
 
         /// A cyphertext encrypted to Alice & Bob, signed by Alice.
         static ref CTEXT_SIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, Some(&KEYS.alice_secret)).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
         };
 
         /// A cyphertext encrypted to Alice & Bob, not signed.
         static ref CTEXT_UNSIGNED: String = {
-            let mut keyring = Keyring::default();
-            keyring.add_owned(KEYS.alice_public.clone());
-            keyring.add_ref(&KEYS.bob_public);
-            pk_encrypt(CLEARTEXT, &keyring, None).unwrap()
+            let mut keyring = Keyring::new();
+            keyring.add(KEYS.alice_public.clone());
+            keyring.add(KEYS.bob_public.clone());
+            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
         };
     }
 
@@ -463,110 +480,115 @@ mod tests {
         assert!(CTEXT_UNSIGNED.starts_with("-----BEGIN PGP MESSAGE-----"));
     }
 
-    #[test]
-    fn test_decrypt_singed() {
+    #[async_std::test]
+    async fn test_decrypt_singed() {
         // Check decrypting as Alice
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring: Keyring<SignedSecretKey> = Keyring::new();
+        decrypt_keyring.add(KEYS.alice_secret.clone());
+        let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
 
         // Check decrypting as Bob
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
     }
 
-    #[test]
-    fn test_decrypt_no_sig_check() {
-        let mut keyring = Keyring::default();
-        keyring.add_ref(&KEYS.alice_secret);
-        let empty_keyring = Keyring::default();
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_no_sig_check() {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_secret.clone());
+        let empty_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &keyring,
-            &empty_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            keyring,
+            empty_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_key() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_key() {
         // The validation does not have the public key of the signer.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.bob_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.bob_public.clone());
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_unsigned() {
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let sig_check_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.alice_public);
-        let mut valid_signatures: HashSet<String> = Default::default();
+    #[async_std::test]
+    async fn test_decrypt_unsigned() {
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let sig_check_keyring = Keyring::new();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
-            CTEXT_UNSIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_UNSIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             Some(&mut valid_signatures),
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
 
-    #[test]
-    fn test_decrypt_signed_no_sigret() {
+    #[async_std::test]
+    async fn test_decrypt_signed_no_sigret() {
         // Check decrypting signed cyphertext without providing the HashSet for signatures.
-        let mut decrypt_keyring = Keyring::default();
-        decrypt_keyring.add_ref(&KEYS.bob_secret);
-        let mut sig_check_keyring = Keyring::default();
-        sig_check_keyring.add_ref(&KEYS.alice_public);
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
         let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes(),
-            &decrypt_keyring,
-            &sig_check_keyring,
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
             None,
         )
+        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
     }

src/provider/data.rs

@@ -19,6 +19,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // aol.md: aol.com
@@ -30,6 +32,23 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // arcor.de.md: arcor.de
+    static ref P_ARCOR_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/arcor-de",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.arcor.de", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.arcor.de", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // autistici.org.md: autistici.org
@@ -43,6 +62,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -56,16 +77,81 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // chello.at.md: chello.at
+    static ref P_CHELLO_AT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/chello-at",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "mail.mymagenta.at", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.mymagenta.at", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_COMCAST: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/comcast",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // dismail.de.md: dismail.de
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISMAIL_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dismail-de",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // disroot.md: disroot.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISROOT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/disroot",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
+
+    // dubby.org.md: dubby.org
+    static ref P_DUBBY_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dubby-org",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // example.com.md: example.com, example.org
     static ref P_EXAMPLE_COM: Provider = Provider {
@@ -78,6 +164,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // fastmail.md: fastmail.com
@@ -89,6 +177,26 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // five.chat.md: five.chat
+    static ref P_FIVE_CHAT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/five-chat",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // freenet.de.md: freenet.de
@@ -102,6 +210,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -115,6 +225,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -129,10 +241,35 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // hey.com.md: hey.com
+    static ref P_HEY_COM: Provider = Provider {
+        status: Status::BROKEN,
+        before_login_hint: "hey.com does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to hey.com.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/hey-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // i.ua.md: i.ua
-    // - skipping provider with status OK and no special things to do
+    static ref P_I_UA: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/i-ua",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // icloud.md: icloud.com, me.com, mac.com
     static ref P_ICLOUD: Provider = Provider {
@@ -145,19 +282,61 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // kolst.com.md: kolst.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KOLST_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kolst-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // kontent.com.md: kontent.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KONTENT_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kontent-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAIL_RU: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mail-ru",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAILBOX_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mailbox-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // nauta.cu.md: nauta.cu
     static ref P_NAUTA_CU: Provider = Provider {
@@ -178,6 +357,8 @@ lazy_static::lazy_static! {
             ConfigDefault { key: Config::E2eeEnabled, value: "0" },
             ConfigDefault { key: Config::MediaQuality, value: "1" },
         ]),
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -191,9 +372,11 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // posteo.md: posteo.de
+    // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
     static ref P_POSTEO: Provider = Provider {
         status: Status::OK,
         before_login_hint: "",
@@ -204,6 +387,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -215,16 +400,48 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // riseup.net.md: riseup.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_RISEUP_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/riseup-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // rogers.com.md: rogers.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_ROGERS_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/rogers-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // systemli.org.md: systemli.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_SYSTEMLI_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/systemli-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
@@ -235,6 +452,8 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // testrun.md: testrun.org
@@ -248,7 +467,14 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // tiscali.it.md: tiscali.it
@@ -262,13 +488,35 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // ukr.net.md: ukr.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_UKR_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/ukr-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // vfemail.md: vfemail.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_VFEMAIL: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/vfemail",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
     static ref P_WEB_DE: Provider = Provider {
@@ -282,12 +530,14 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
     static ref P_YAHOO: Provider = Provider {
         status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with your Yahoo email address you have to allow \"less secure apps\" in the Yahoo webinterface.",
+        before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yahoo",
         server: vec![
@@ -295,9 +545,11 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // yandex.ru.md: yandex.ru, yandex.com
+    // yandex.ru.md: yandex.com, yandex.by, yandex.kz, yandex.ru, yandex.ua, ya.ru, narod.ru
     static ref P_YANDEX_RU: Provider = Provider {
         status: Status::PREPARATION,
         before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
@@ -306,6 +558,8 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -319,16 +573,26 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
+        ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("chello.at", &*P_CHELLO_AT),
+        ("xfinity.com", &*P_COMCAST),
+        ("comcast.net", &*P_COMCAST),
+        ("dismail.de", &*P_DISMAIL_DE),
+        ("disroot.org", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -341,9 +605,19 @@ lazy_static::lazy_static! {
         ("gmx.info", &*P_GMX_NET),
         ("gmx.biz", &*P_GMX_NET),
         ("gmx.com", &*P_GMX_NET),
+        ("hey.com", &*P_HEY_COM),
+        ("i.ua", &*P_I_UA),
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
+        ("kolst.com", &*P_KOLST_COM),
+        ("kontent.com", &*P_KONTENT_COM),
+        ("mail.ru", &*P_MAIL_RU),
+        ("inbox.ru", &*P_MAIL_RU),
+        ("bk.ru", &*P_MAIL_RU),
+        ("list.ru", &*P_MAIL_RU),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
@@ -351,12 +625,65 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
+        ("posteo.af", &*P_POSTEO),
+        ("posteo.at", &*P_POSTEO),
+        ("posteo.be", &*P_POSTEO),
+        ("posteo.ch", &*P_POSTEO),
+        ("posteo.cl", &*P_POSTEO),
+        ("posteo.co", &*P_POSTEO),
+        ("posteo.co.uk", &*P_POSTEO),
+        ("posteo.com.br", &*P_POSTEO),
+        ("posteo.cr", &*P_POSTEO),
+        ("posteo.cz", &*P_POSTEO),
+        ("posteo.dk", &*P_POSTEO),
+        ("posteo.ee", &*P_POSTEO),
+        ("posteo.es", &*P_POSTEO),
+        ("posteo.eu", &*P_POSTEO),
+        ("posteo.fi", &*P_POSTEO),
+        ("posteo.gl", &*P_POSTEO),
+        ("posteo.gr", &*P_POSTEO),
+        ("posteo.hn", &*P_POSTEO),
+        ("posteo.hr", &*P_POSTEO),
+        ("posteo.hu", &*P_POSTEO),
+        ("posteo.ie", &*P_POSTEO),
+        ("posteo.in", &*P_POSTEO),
+        ("posteo.is", &*P_POSTEO),
+        ("posteo.jp", &*P_POSTEO),
+        ("posteo.la", &*P_POSTEO),
+        ("posteo.li", &*P_POSTEO),
+        ("posteo.lt", &*P_POSTEO),
+        ("posteo.lu", &*P_POSTEO),
+        ("posteo.me", &*P_POSTEO),
+        ("posteo.mx", &*P_POSTEO),
+        ("posteo.my", &*P_POSTEO),
+        ("posteo.net", &*P_POSTEO),
+        ("posteo.nl", &*P_POSTEO),
+        ("posteo.no", &*P_POSTEO),
+        ("posteo.nz", &*P_POSTEO),
+        ("posteo.org", &*P_POSTEO),
+        ("posteo.pe", &*P_POSTEO),
+        ("posteo.pl", &*P_POSTEO),
+        ("posteo.pm", &*P_POSTEO),
+        ("posteo.pt", &*P_POSTEO),
+        ("posteo.ro", &*P_POSTEO),
+        ("posteo.ru", &*P_POSTEO),
+        ("posteo.se", &*P_POSTEO),
+        ("posteo.sg", &*P_POSTEO),
+        ("posteo.si", &*P_POSTEO),
+        ("posteo.tn", &*P_POSTEO),
+        ("posteo.uk", &*P_POSTEO),
+        ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("rogers.com", &*P_ROGERS_COM),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
+        ("ukr.net", &*P_UKR_NET),
+        ("vfemail.net", &*P_VFEMAIL),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),
         ("flirt.ms", &*P_WEB_DE),
@@ -399,8 +726,13 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
-        ("yandex.ru", &*P_YANDEX_RU),
         ("yandex.com", &*P_YANDEX_RU),
+        ("yandex.by", &*P_YANDEX_RU),
+        ("yandex.kz", &*P_YANDEX_RU),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("yandex.ua", &*P_YANDEX_RU),
+        ("ya.ru", &*P_YANDEX_RU),
+        ("narod.ru", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/provider/mod.rs

@@ -35,6 +35,13 @@ pub enum UsernamePattern {
     EMAILLOCALPART = 2,
 }
 
+#[derive(Debug, PartialEq)]
+#[repr(u8)]
+pub enum Oauth2Authorizer {
+    Yandex = 1,
+    Gmail = 2,
+}
+
 #[derive(Debug)]
 pub struct Server {
     pub protocol: Protocol,
@@ -72,6 +79,8 @@ pub struct Provider {
     pub overview_page: &'static str,
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
+    pub strict_tls: bool,
+    pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
 impl Provider {

src/provider/update.py

@@ -100,6 +100,12 @@ def process_data(data, file):
 
     config_defaults = process_config_defaults(data)
 
+    strict_tls = data.get("strict_tls", False)
+    strict_tls = "true" if strict_tls else "false"
+
+    oauth2 = data.get("oauth2", "")
+    oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -111,6 +117,8 @@ def process_data(data, file):
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
         provider += "        config_defaults: " + config_defaults + ",\n"
+        provider += "        strict_tls: " + strict_tls + ",\n"
+        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -121,11 +129,11 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
-        out_all += "    // - skipping provider with status OK and no special things to do\n\n"
-    else:
-        out_all += provider
-        out_domains += domains
+
+    # also add provider with no special things to do -
+    # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
+    out_all += provider
+    out_domains += domains
 
 
 def process_file(file):

src/qr.rs

@@ -10,8 +10,7 @@ use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::key::dc_format_fingerprint;
-use crate::key::dc_normalize_fingerprint;
+use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
@@ -80,6 +79,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         Some(pair) => pair,
         None => (payload, ""),
     };
+    let fingerprint: Fingerprint = match fingerprint.parse() {
+        Ok(fp) => fp,
+        Err(err) => {
+            return Error::new(err)
+                .context("Failed to parse fingerprint in QR code")
+                .into()
+        }
+    };
 
     // replace & with \n to match expected param format
     let fragment = fragment.replace('&', "\n");
@@ -128,13 +135,6 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         None
     };
 
-    let fingerprint = dc_normalize_fingerprint(fingerprint);
-
-    // ensure valid fingerprint
-    if fingerprint.len() != 40 {
-        return format_err!("Bad fingerprint length in QR code").into();
-    }
-
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
@@ -161,7 +161,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr)).await;
         } else {
             lot.state = LotState::QrFprWithoutAddr;
-            lot.text1 = Some(dc_format_fingerprint(&fingerprint));
+            lot.text1 = Some(fingerprint.to_string());
         }
     } else if let Some(addr) = addr {
         if grpid.is_some() && grpname.is_some() {
@@ -383,11 +383,11 @@ fn normalize_address(addr: &str) -> Result<String, Error> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_decode_http() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "http://www.hello.com").await;
 
@@ -399,7 +399,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_https() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "https://www.hello.com").await;
 
@@ -411,7 +411,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_text() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "I am so cool").await;
 
@@ -423,7 +423,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_vcard() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -441,7 +441,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_matmsg() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -459,7 +459,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_mailto() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -485,7 +485,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_smtp() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld").await;
 
@@ -499,7 +499,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_group() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -528,7 +528,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_secure_join() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -556,7 +556,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_without_addr() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -591,7 +591,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_account() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -613,7 +613,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_account_bad_scheme() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
             "DCACCOUNT:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",

src/scheduler.rs

@@ -1,13 +1,13 @@
+#![warn(clippy::indexing_slicing)]
+
 use async_std::prelude::*;
 use async_std::sync::{channel, Receiver, Sender};
 use async_std::task;
 
-use std::time::Duration;
-
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::smtp::Smtp;
+use crate::{config::Config, message::MsgId, smtp::Smtp};
 
 pub(crate) struct StopToken;
 
@@ -25,30 +25,21 @@ pub(crate) enum Scheduler {
         sentbox_handle: Option<task::JoinHandle<()>>,
         smtp: SmtpConnectionState,
         smtp_handle: Option<task::JoinHandle<()>>,
-        probe_network: bool,
     },
 }
 
 impl Context {
     /// Indicate that the network likely has come back.
     pub async fn maybe_network(&self) {
-        self.scheduler.write().await.maybe_network().await;
+        self.scheduler.read().await.maybe_network().await;
     }
 
-    pub(crate) async fn interrupt_inbox(&self) {
-        self.scheduler.read().await.interrupt_inbox().await;
+    pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_inbox(info).await;
     }
 
-    pub(crate) async fn interrupt_sentbox(&self) {
-        self.scheduler.read().await.interrupt_sentbox().await;
-    }
-
-    pub(crate) async fn interrupt_mvbox(&self) {
-        self.scheduler.read().await.interrupt_mvbox().await;
-    }
-
-    pub(crate) async fn interrupt_smtp(&self) {
-        self.scheduler.read().await.interrupt_smtp().await;
+    pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_smtp(info).await;
     }
 }
 
@@ -66,33 +57,30 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         // track number of continously executed jobs
         let mut jobs_loaded = 0;
+        let mut info = InterruptInfo::default();
         loop {
-            let probe_network = ctx.scheduler.read().await.get_probe_network();
-            match job::load_next(&ctx, Thread::Imap, probe_network)
-                .timeout(Duration::from_millis(200))
-                .await
-            {
-                Ok(Some(job)) if jobs_loaded <= 20 => {
+            match job::load_next(&ctx, Thread::Imap, &info).await {
+                Some(job) if jobs_loaded <= 20 => {
                     jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
-                    ctx.scheduler.write().await.set_probe_network(false);
+                    info = Default::default();
                 }
-                Ok(Some(job)) => {
+                Some(job) => {
                     // Let the fetch run, but return back to the job afterwards.
                     info!(ctx, "postponing imap-job {} to run fetch...", job);
                     jobs_loaded = 0;
                     fetch(&ctx, &mut connection).await;
                 }
-                Ok(None) | Err(async_std::future::TimeoutError { .. }) => {
+                None => {
                     jobs_loaded = 0;
-                    fetch_idle(&ctx, &mut connection).await;
+                    info = if ctx.get_config_bool(Config::InboxWatch).await {
+                        fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
+                    } else {
+                        connection.fake_idle(&ctx, None).await
+                    };
                 }
             }
         }
@@ -109,15 +97,18 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 }
 
 async fn fetch(ctx: &Context, connection: &mut Imap) {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "{}", err);
+                return;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
         }
         None => {
             warn!(ctx, "Can not fetch inbox folder, not set");
@@ -126,16 +117,20 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     }
 }
 
-async fn fetch_idle(ctx: &Context, connection: &mut Imap) {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
+    match ctx.get_config(folder).await {
         Some(watch_folder) => {
+            // connect and fake idle if unable to connect
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "imap connection failed: {}", err);
+                return connection.fake_idle(&ctx, None).await;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
 
             // idle
             if connection.can_idle() {
@@ -143,15 +138,17 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap) {
                     .idle(&ctx, Some(watch_folder))
                     .await
                     .unwrap_or_else(|err| {
-                        error!(ctx, "{}", err);
-                    });
+                        connection.trigger_reconnect();
+                        warn!(ctx, "{}", err);
+                        InterruptInfo::new(false, None)
+                    })
             } else {
-                connection.fake_idle(&ctx, Some(watch_folder)).await;
+                connection.fake_idle(&ctx, Some(watch_folder)).await
             }
         }
         None => {
-            warn!(ctx, "Can not watch inbox folder, not set");
-            connection.fake_idle(&ctx, None).await;
+            warn!(ctx, "Can not watch {} folder, not set", folder);
+            connection.fake_idle(&ctx, None).await
         }
     }
 }
@@ -160,7 +157,7 @@ async fn simple_imap_loop(
     ctx: Context,
     started: Sender<()>,
     inbox_handlers: ImapConnectionHandlers,
-    folder: impl AsRef<str>,
+    folder: Config,
 ) {
     use futures::future::FutureExt;
 
@@ -176,43 +173,9 @@ async fn simple_imap_loop(
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         loop {
-            match get_watch_folder(&ctx, folder.as_ref()).await {
-                Some(watch_folder) => {
-                    // fetch
-                    connection
-                        .fetch(&ctx, &watch_folder)
-                        .await
-                        .unwrap_or_else(|err| {
-                            error!(ctx, "{}", err);
-                        });
-
-                    // idle
-                    if connection.can_idle() {
-                        connection
-                            .idle(&ctx, Some(watch_folder))
-                            .await
-                            .unwrap_or_else(|err| {
-                                error!(ctx, "{}", err);
-                            });
-                    } else {
-                        connection.fake_idle(&ctx, Some(watch_folder)).await;
-                    }
-                }
-                None => {
-                    warn!(
-                        &ctx,
-                        "No watch folder found for {}, skipping",
-                        folder.as_ref()
-                    );
-                    connection.fake_idle(&ctx, None).await
-                }
-            }
+            fetch_idle(&ctx, &mut connection, folder).await;
         }
     };
 
@@ -241,25 +204,20 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
+
+        let mut interrupt_info = Default::default();
         loop {
-            let probe_network = ctx.scheduler.read().await.get_probe_network();
-            match job::load_next(&ctx, Thread::Smtp, probe_network)
-                .timeout(Duration::from_millis(200))
-                .await
-            {
-                Ok(Some(job)) => {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
+                Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
-                    ctx.scheduler.write().await.set_probe_network(false);
+                    interrupt_info = Default::default();
                 }
-                Ok(None) | Err(async_std::future::TimeoutError { .. }) => {
-                    info!(ctx, "smtp fake idle");
+                None => {
                     // Fake Idle
-                    idle_interrupt_receiver
-                        .recv()
-                        .timeout(Duration::from_secs(5))
-                        .await
-                        .ok();
+                    info!(ctx, "smtp fake idle - started");
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    info!(ctx, "smtp fake idle - interrupted")
                 }
             }
         }
@@ -283,62 +241,64 @@ impl Scheduler {
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
         let (inbox, inbox_handlers) = ImapConnectionState::new();
 
+        let (inbox_start_send, inbox_start_recv) = channel(1);
+        let (mvbox_start_send, mvbox_start_recv) = channel(1);
+        let mut mvbox_handle = None;
+        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let mut sentbox_handle = None;
+        let (smtp_start_send, smtp_start_recv) = channel(1);
+
+        let ctx1 = ctx.clone();
+        let inbox_handle = Some(task::spawn(async move {
+            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
+        }));
+
+        if ctx.get_config_bool(Config::MvboxWatch).await {
+            let ctx1 = ctx.clone();
+            mvbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    mvbox_start_send,
+                    mvbox_handlers,
+                    Config::ConfiguredMvboxFolder,
+                )
+                .await
+            }));
+        } else {
+            mvbox_start_send.send(()).await;
+        }
+
+        if ctx.get_config_bool(Config::SentboxWatch).await {
+            let ctx1 = ctx.clone();
+            sentbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    sentbox_start_send,
+                    sentbox_handlers,
+                    Config::ConfiguredSentboxFolder,
+                )
+                .await
+            }));
+        } else {
+            sentbox_start_send.send(()).await;
+        }
+
+        let ctx1 = ctx.clone();
+        let smtp_handle = Some(task::spawn(async move {
+            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
+        }));
+
         *self = Scheduler::Running {
             inbox,
             mvbox,
             sentbox,
             smtp,
-            probe_network: false,
-            inbox_handle: None,
-            mvbox_handle: None,
-            sentbox_handle: None,
-            smtp_handle: None,
+            inbox_handle,
+            mvbox_handle,
+            sentbox_handle,
+            smtp_handle,
         };
 
-        let (inbox_start_send, inbox_start_recv) = channel(1);
-        if let Scheduler::Running { inbox_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *inbox_handle = Some(task::spawn(async move {
-                inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
-            }));
-        }
-
-        let (mvbox_start_send, mvbox_start_recv) = channel(1);
-        if let Scheduler::Running { mvbox_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *mvbox_handle = Some(task::spawn(async move {
-                simple_imap_loop(
-                    ctx1,
-                    mvbox_start_send,
-                    mvbox_handlers,
-                    "configured_mvbox_folder",
-                )
-                .await
-            }));
-        }
-
-        let (sentbox_start_send, sentbox_start_recv) = channel(1);
-        if let Scheduler::Running { sentbox_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *sentbox_handle = Some(task::spawn(async move {
-                simple_imap_loop(
-                    ctx1,
-                    sentbox_start_send,
-                    sentbox_handlers,
-                    "configured_sentbox_folder",
-                )
-                .await
-            }));
-        }
-
-        let (smtp_start_send, smtp_start_recv) = channel(1);
-        if let Scheduler::Running { smtp_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *smtp_handle = Some(task::spawn(async move {
-                smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
-            }));
-        }
-
         // wait for all loops to be started
         if let Err(err) = inbox_start_recv
             .recv()
@@ -353,58 +313,39 @@ impl Scheduler {
         info!(ctx, "scheduler is running");
     }
 
-    fn set_probe_network(&mut self, val: bool) {
-        match self {
-            Scheduler::Running {
-                ref mut probe_network,
-                ..
-            } => {
-                *probe_network = val;
-            }
-            _ => panic!("set_probe_network can only be called when running"),
-        }
-    }
-
-    fn get_probe_network(&self) -> bool {
-        match self {
-            Scheduler::Running { probe_network, .. } => *probe_network,
-            _ => panic!("get_probe_network can only be called when running"),
-        }
-    }
-
-    async fn maybe_network(&mut self) {
+    async fn maybe_network(&self) {
         if !self.is_running() {
             return;
         }
-        self.set_probe_network(true);
-        self.interrupt_inbox()
-            .join(self.interrupt_mvbox())
-            .join(self.interrupt_sentbox())
-            .join(self.interrupt_smtp())
+
+        self.interrupt_inbox(InterruptInfo::new(true, None))
+            .join(self.interrupt_mvbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_smtp(InterruptInfo::new(true, None)))
             .await;
     }
 
-    async fn interrupt_inbox(&self) {
+    async fn interrupt_inbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref inbox, .. } = self {
-            inbox.interrupt().await;
+            inbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_mvbox(&self) {
+    async fn interrupt_mvbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref mvbox, .. } = self {
-            mvbox.interrupt().await;
+            mvbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_sentbox(&self) {
+    async fn interrupt_sentbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref sentbox, .. } = self {
-            sentbox.interrupt().await;
+            sentbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_smtp(&self) {
+    async fn interrupt_smtp(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref smtp, .. } = self {
-            smtp.interrupt().await;
+            smtp.interrupt(info).await;
         }
     }
 
@@ -446,10 +387,18 @@ impl Scheduler {
                 smtp_handle,
                 ..
             } => {
-                inbox_handle.take().expect("inbox not started").await;
-                mvbox_handle.take().expect("mvbox not started").await;
-                sentbox_handle.take().expect("sentbox not started").await;
-                smtp_handle.take().expect("smtp not started").await;
+                if let Some(handle) = inbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = mvbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = sentbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = smtp_handle.take() {
+                    handle.await;
+                }
 
                 *self = Scheduler::Stopped;
             }
@@ -473,7 +422,7 @@ struct ConnectionState {
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
-    idle_interrupt_sender: Sender<()>,
+    idle_interrupt_sender: Sender<InterruptInfo>,
 }
 
 impl ConnectionState {
@@ -485,11 +434,9 @@ impl ConnectionState {
         self.shutdown_receiver.recv().await.ok();
     }
 
-    async fn interrupt(&self) {
-        if !self.idle_interrupt_sender.is_full() {
-            // Use try_send to avoid blocking on interrupts.
-            self.idle_interrupt_sender.send(()).await;
-        }
+    async fn interrupt(&self, info: InterruptInfo) {
+        // Use try_send to avoid blocking on interrupts.
+        self.idle_interrupt_sender.try_send(info).ok();
     }
 }
 
@@ -523,8 +470,8 @@ impl SmtpConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self) {
-        self.state.interrupt().await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -533,12 +480,11 @@ impl SmtpConnectionState {
     }
 }
 
-#[derive(Debug)]
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
     shutdown_sender: Sender<()>,
-    idle_interrupt_receiver: Receiver<()>,
+    idle_interrupt_receiver: Receiver<InterruptInfo>,
 }
 
 #[derive(Debug)]
@@ -550,8 +496,8 @@ impl ImapConnectionState {
     /// Construct a new connection.
     fn new() -> (Self, ImapConnectionHandlers) {
         let (stop_sender, stop_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
         let (shutdown_sender, shutdown_receiver) = channel(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new(idle_interrupt_receiver),
@@ -571,8 +517,8 @@ impl ImapConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self) {
-        self.state.interrupt().await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -588,20 +534,17 @@ struct ImapConnectionHandlers {
     shutdown_sender: Sender<()>,
 }
 
-async fn get_watch_folder(context: &Context, config_name: impl AsRef<str>) -> Option<String> {
-    match context
-        .sql
-        .get_raw_config(context, config_name.as_ref())
-        .await
-    {
-        Some(name) => Some(name),
-        None => {
-            if config_name.as_ref() == "configured_inbox_folder" {
-                // initialized with old version, so has not set configured_inbox_folder
-                Some("INBOX".to_string())
-            } else {
-                None
-            }
+#[derive(Default, Debug)]
+pub struct InterruptInfo {
+    pub probe_network: bool,
+    pub msg_id: Option<MsgId>,
+}
+
+impl InterruptInfo {
+    pub fn new(probe_network: bool, msg_id: Option<MsgId>) -> Self {
+        Self {
+            probe_network,
+            msg_id,
         }
     }
 }

src/securejoin.rs

@@ -14,7 +14,7 @@ use crate::e2ee::*;
 use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, DcKey, Key, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -73,8 +73,6 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     ====   Step 1 in "Setup verified contact" protocol   ====
     =======================================================*/
 
-    let fingerprint: String;
-
     ensure_secret_key_exists(context).await.ok();
 
     // invitenumber will be used to allow starting the handshake,
@@ -95,7 +93,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         .await
         .unwrap_or_default();
 
-    fingerprint = match get_self_fingerprint(context).await {
+    let fingerprint: Fingerprint = match get_self_fingerprint(context).await {
         Some(fp) => fp,
         None => {
             return None;
@@ -116,7 +114,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
 
             Some(format!(
                 "OPENPGP4FPR:{}#a={}&g={}&x={}&i={}&s={}",
-                fingerprint,
+                fingerprint.hex(),
                 self_addr_urlencoded,
                 &group_name_urlencoded,
                 &chat.grpid,
@@ -131,7 +129,11 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         // parameters used: a=n=i=s=
         Some(format!(
             "OPENPGP4FPR:{}#a={}&n={}&i={}&s={}",
-            fingerprint, self_addr_urlencoded, self_name_urlencoded, &invitenumber, &auth,
+            fingerprint.hex(),
+            self_addr_urlencoded,
+            self_name_urlencoded,
+            &invitenumber,
+            &auth,
         ))
     };
 
@@ -140,9 +142,9 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     qr
 }
 
-async fn get_self_fingerprint(context: &Context) -> Option<String> {
+async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
     match SignedPublicKey::load_self(context).await {
-        Ok(key) => Some(Key::from(key).fingerprint()),
+        Ok(key) => Some(key.fingerprint()),
         Err(_) => {
             warn!(context, "get_self_fingerprint(): failed to load key");
             None
@@ -249,7 +251,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
             chat_id_2_contact_id(context, contact_chat_id).await,
             400
         );
-        let own_fingerprint = get_self_fingerprint(context).await.unwrap_or_default();
+        let own_fingerprint = get_self_fingerprint(context).await;
 
         // Bob -> Alice
         if let Err(err) = send_handshake_msg(
@@ -261,7 +263,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
                 "vc-request-with-auth"
             },
             get_qr_attr!(context, auth).to_string(),
-            Some(own_fingerprint),
+            own_fingerprint,
             if join_vg {
                 get_qr_attr!(context, text2).to_string()
             } else {
@@ -311,7 +313,7 @@ async fn send_handshake_msg(
     contact_chat_id: ChatId,
     step: &str,
     param2: impl AsRef<str>,
-    fingerprint: Option<String>,
+    fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
 ) -> Result<(), HandshakeError> {
     let mut msg = Message::default();
@@ -328,7 +330,7 @@ async fn send_handshake_msg(
         msg.param.set(Param::Arg2, param2);
     }
     if let Some(fp) = fingerprint {
-        msg.param.set(Param::Arg3, fp);
+        msg.param.set(Param::Arg3, fp.hex());
     }
     if !grpid.as_ref().is_empty() {
         msg.param.set(Param::Arg4, grpid.as_ref());
@@ -360,7 +362,7 @@ async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32
 
 async fn fingerprint_equals_sender(
     context: &Context,
-    fingerprint: impl AsRef<str>,
+    fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
 ) -> bool {
     let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
@@ -368,9 +370,8 @@ async fn fingerprint_equals_sender(
     if contacts.len() == 1 {
         if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
             if let Some(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
-                let fingerprint_normalized = dc_normalize_fingerprint(fingerprint.as_ref());
                 if peerstate.public_key_fingerprint.is_some()
-                    && &fingerprint_normalized == peerstate.public_key_fingerprint.as_ref().unwrap()
+                    && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
                     return true;
                 }
@@ -397,6 +398,8 @@ pub(crate) enum HandshakeError {
     NoSelfAddr,
     #[error("Failed to send message")]
     MsgSendFailed(#[source] Error),
+    #[error("Failed to parse fingerprint")]
+    BadFingerprint(#[from] crate::key::FingerprintError),
 }
 
 /// What to do with a Secure-Join handshake message after it was handled.
@@ -516,10 +519,11 @@ pub(crate) async fn handle_securejoin_handshake(
                 // no error, just aborted somehow or a mail from another handshake
                 return Ok(HandshakeMessage::Ignore);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
             let auth = get_qr_attr!(context, auth).to_string();
 
-            if !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice) {
+            if !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -576,8 +580,9 @@ pub(crate) async fn handle_securejoin_handshake(
             ==========================================================*/
 
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -588,7 +593,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !encrypted_and_signed(context, mime_message, &fingerprint) {
+            if !encrypted_and_signed(context, mime_message, Some(&fingerprint)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -625,7 +630,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -673,7 +678,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     contact_chat_id,
                     "vc-contact-confirm",
                     "",
-                    Some(fingerprint.clone()),
+                    Some(fingerprint),
                     "",
                 )
                 .await?;
@@ -709,7 +714,8 @@ pub(crate) async fn handle_securejoin_handshake(
                 );
                 return Ok(abort_retval);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
@@ -731,7 +737,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 true
             };
             if vg_expect_encrypted
-                && !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice)
+                && !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice))
             {
                 could_not_establish_secure_connection(
                     context,
@@ -888,7 +894,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
             if !encrypted_and_signed(
                 context,
                 mime_message,
-                get_self_fingerprint(context).await.unwrap_or_default(),
+                get_self_fingerprint(context).await.as_ref(),
             ) {
                 could_not_establish_secure_connection(
                     context,
@@ -898,8 +904,9 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -910,7 +917,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -967,16 +974,13 @@ async fn could_not_establish_secure_connection(
     error!(context, "{} ({})", &msg, details);
 }
 
-async fn mark_peer_as_verified(
-    context: &Context,
-    fingerprint: impl AsRef<str>,
-) -> Result<(), Error> {
+async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
     if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint.as_ref()).await
+        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await
     {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
-            fingerprint.as_ref(),
+            fingerprint,
             PeerstateVerifiedStatus::BidirectVerified,
         ) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
@@ -990,7 +994,7 @@ async fn mark_peer_as_verified(
     }
     bail!(
         "could not mark peer as verified for fingerprint {}",
-        fingerprint.as_ref()
+        fingerprint.hex()
     );
 }
 
@@ -1001,7 +1005,7 @@ async fn mark_peer_as_verified(
 fn encrypted_and_signed(
     context: &Context,
     mimeparser: &MimeMessage,
-    expected_fingerprint: impl AsRef<str>,
+    expected_fingerprint: Option<&Fingerprint>,
 ) -> bool {
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
@@ -1009,17 +1013,17 @@ fn encrypted_and_signed(
     } else if mimeparser.signatures.is_empty() {
         warn!(context, "Message not signed.",);
         false
-    } else if expected_fingerprint.as_ref().is_empty() {
-        warn!(context, "Fingerprint for comparison missing.",);
+    } else if expected_fingerprint.is_none() {
+        warn!(context, "Fingerprint for comparison missing.");
         false
     } else if !mimeparser
         .signatures
-        .contains(expected_fingerprint.as_ref())
+        .contains(expected_fingerprint.unwrap())
     {
         warn!(
             context,
             "Message does not match expected fingerprint {}.",
-            expected_fingerprint.as_ref(),
+            expected_fingerprint.unwrap(),
         );
         false
     } else {

src/smtp/mod.rs

@@ -1,5 +1,7 @@
 //! # SMTP transport module
 
+#![forbid(clippy::indexing_slicing)]
+
 pub mod send;
 
 use std::time::{Duration, Instant};
@@ -10,8 +12,9 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
-use crate::login_param::{dc_build_tls, LoginParam};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::oauth2::*;
+use crate::provider::get_provider_info;
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -44,9 +47,8 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Default, DebugStub)]
-pub struct Smtp {
-    #[debug_stub(some = "SmtpTransport")]
+#[derive(Default)]
+pub(crate) struct Smtp {
     transport: Option<smtp::SmtpTransport>,
 
     /// Email address we are sending from.
@@ -113,7 +115,14 @@ impl Smtp {
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls_config = dc_build_tls(lp.smtp_certificate_checks);
+        let provider = get_provider_info(&lp.addr);
+        let strict_tls = match lp.smtp_certificate_checks {
+            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Strict => true,
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => false,
+        };
+        let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
@@ -172,7 +181,7 @@ impl Smtp {
                 .stock_string_repl_str2(
                     StockMessage::ServerResponse,
                     format!("SMTP {}:{}", domain, port),
-                    err.to_string(),
+                    format!("{}, ({:?})", err.to_string(), err),
                 )
                 .await;
 

src/sql.rs

@@ -13,6 +13,7 @@ use crate::chat::{update_device_icon, update_saved_messages_icon};
 use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::start_ephemeral_timers;
 use crate::param::*;
 use crate::peerstate::*;
 
@@ -49,7 +50,7 @@ pub enum Error {
 pub type Result<T> = std::result::Result<T, Error>;
 
 /// A wrapper around the underlying Sqlite3 object.
-#[derive(DebugStub)]
+#[derive(Debug)]
 pub struct Sql {
     pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
 }
@@ -568,10 +569,17 @@ pub async fn housekeeping(context: &Context) {
         }
     }
 
+    if let Err(err) = start_ephemeral_timers(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot start ephemeral timers: {}", err
+        );
+    }
+
     if let Err(err) = prune_tombstones(context).await {
         warn!(
             context,
-            "Houskeeping: Cannot prune message tombstones: {}", err
+            "Housekeeping: Cannot prune message tombstones: {}", err
         );
     }
 
@@ -593,7 +601,7 @@ fn is_file_in_use(files_in_use: &HashSet<String>, namespc_opt: Option<&str>, nam
 }
 
 fn maybe_add_file(files_in_use: &mut HashSet<String>, file: impl AsRef<str>) {
-    if !file.as_ref().starts_with("$BLOBDIR") {
+    if !file.as_ref().starts_with("$BLOBDIR/") {
         return;
     }
 
@@ -1241,6 +1249,41 @@ async fn open(
                 .await?;
             sql.set_raw_config_int(context, "dbversion", 63).await?;
         }
+        if dbversion < 64 {
+            info!(context, "[migration] v64");
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN error TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 64).await?;
+        }
+        if dbversion < 65 {
+            info!(context, "[migration] v65");
+            sql.execute(
+                "ALTER TABLE chats ADD COLUMN ephemeral_timer INTEGER",
+                paramsv![],
+            )
+            .await?;
+            // Timer value in seconds. For incoming messages this
+            // timer starts when message is read, so we want to have
+            // the value stored here until the timer starts.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            // Timestamp indicating when the message should be
+            // deleted. It is convenient to store it here because UI
+            // needs this value to display how much time is left until
+            // the message is deleted.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 65).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1303,10 +1346,12 @@ mod test {
         maybe_add_file(&mut files, "$BLOBDIR/hello");
         maybe_add_file(&mut files, "$BLOBDIR/world.txt");
         maybe_add_file(&mut files, "world2.txt");
+        maybe_add_file(&mut files, "$BLOBDIR");
 
         assert!(files.contains("hello"));
         assert!(files.contains("world.txt"));
         assert!(!files.contains("world2.txt"));
+        assert!(!files.contains("$BLOBDIR"));
     }
 
     #[test]

src/stock.rs

@@ -177,8 +177,37 @@ pub enum StockMessage {
                     however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
     WelcomeMessage = 71,
 
-    #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
+    #[strum(props(fallback = "Unknown sender for this chat. See 'info' for more details."))]
     UnknownSenderForChat = 72,
+
+    #[strum(props(fallback = "Message from %1$s"))]
+    SubjectForNewContact = 73,
+
+    #[strum(props(fallback = "Failed to send message to %1$s."))]
+    FailedSendingTo = 74,
+
+    #[strum(props(fallback = "Message deletion timer is disabled."))]
+    MsgEphemeralTimerDisabled = 75,
+
+    // A fallback message for unknown timer values.
+    // "s" stands for "second" SI unit here.
+    #[strum(props(fallback = "Message deletion timer is set to %1$s s."))]
+    MsgEphemeralTimerEnabled = 76,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 minute."))]
+    MsgEphemeralTimerMinute = 77,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 hour."))]
+    MsgEphemeralTimerHour = 78,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 day."))]
+    MsgEphemeralTimerDay = 79,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 week."))]
+    MsgEphemeralTimerWeek = 80,
+
+    #[strum(props(fallback = "Message deletion timer is set to 4 weeks."))]
+    MsgEphemeralTimerFourWeeks = 81,
 }
 
 /*
@@ -328,10 +357,10 @@ impl Context {
         let action1 = action.trim_end_matches('.');
         match from_id {
             0 => action,
-            1 => {
+            DC_CONTACT_ID_SELF => {
                 self.stock_string_repl_str(StockMessage::MsgActionByMe, action1)
                     .await
-            } // DC_CONTACT_ID_SELF
+            }
             _ => {
                 let displayname = Contact::get_by_id(self, from_id)
                     .await
@@ -403,7 +432,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_stock_translation() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::NoMessages, "xyz".to_string())
             .await
@@ -413,7 +442,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_stock_translation_wrong_replacements() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert!(t
             .ctx
             .set_stock_translation(StockMessage::NoMessages, "xyz %1$s ".to_string())
@@ -428,7 +457,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_str() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx.stock_str(StockMessage::NoMessages).await,
             "No messages."
@@ -437,7 +466,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_str() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         // uses %1$s substitution
         assert_eq!(
             t.ctx
@@ -450,7 +479,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_int() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_string_repl_int(StockMessage::MsgAddMember, 42)
@@ -461,7 +490,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_str2() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar")
@@ -472,7 +501,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_simple() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
@@ -483,7 +512,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_me() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(
@@ -499,7 +528,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_me_with_displayname() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         Contact::create(&t.ctx, "Alice", "alice@example.com")
             .await
             .expect("failed to create contact");
@@ -518,7 +547,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_other_with_displayname() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let contact_id = {
             Contact::create(&t.ctx, "Alice", "alice@example.com")
                 .await
@@ -542,7 +571,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_grp_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(
@@ -558,7 +587,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_grp_name_other() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let id = Contact::create(&t.ctx, "Alice", "alice@example.com")
             .await
             .expect("failed to create contact");
@@ -573,7 +602,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_update_device_chats() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.ok();
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 2);

src/test_utils.rs

@@ -18,26 +18,59 @@ pub(crate) struct TestContext {
     pub dir: TempDir,
 }
 
-/// Create a new, opened [TestContext] using given callback.
-///
-/// The [Context] will be opened with the SQLite database named
-/// "db.sqlite" in the [TestContext.dir] directory.
-///
-/// [Context]: crate::context::Context
-pub(crate) async fn test_context() -> TestContext {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let ctx = Context::new("FakeOs".into(), dbfile.into()).await.unwrap();
-    TestContext { ctx, dir }
-}
+impl TestContext {
+    /// Create a new [TestContext].
+    ///
+    /// The [Context] will be created and have an SQLite database named "db.sqlite" in the
+    /// [TestContext.dir] directory.  This directory is cleaned up when the [TestContext] is
+    /// dropped.
+    ///
+    /// [Context]: crate::context::Context
+    pub async fn new() -> Self {
+        let dir = tempdir().unwrap();
+        let dbfile = dir.path().join("db.sqlite");
+        let ctx = Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Self { ctx, dir }
+    }
 
-/// Return a dummy [TestContext].
-///
-/// The context will be opened and use the SQLite database as
-/// specified in [test_context] but there is no callback hooked up,
-/// i.e. [Context::call_cb] will always return `0`.
-pub(crate) async fn dummy_context() -> TestContext {
-    test_context().await
+    /// Create a new configured [TestContext].
+    ///
+    /// This is a shortcut which automatically calls [TestContext::configure_alice] after
+    /// creating the context.
+    pub async fn new_alice() -> Self {
+        let t = Self::new().await;
+        t.configure_alice().await;
+        t
+    }
+
+    /// Configure with alice@example.com.
+    ///
+    /// The context will be fake-configured as the alice user, with a pre-generated secret
+    /// key.  The email address of the user is returned as a string.
+    pub async fn configure_alice(&self) -> String {
+        let keypair = alice_keypair();
+        self.configure_addr(&keypair.addr.to_string()).await;
+        key::store_self_keypair(&self.ctx, &keypair, key::KeyPairUse::Default)
+            .await
+            .expect("Failed to save Alice's key");
+        keypair.addr.to_string()
+    }
+
+    /// Configure as a given email address.
+    ///
+    /// The context will be configured but the key will not be pre-generated so if a key is
+    /// used the fingerprint will be different every time.
+    pub async fn configure_addr(&self, addr: &str) {
+        self.ctx.set_config(Config::Addr, Some(addr)).await.unwrap();
+        self.ctx
+            .set_config(Config::ConfiguredAddr, Some(addr))
+            .await
+            .unwrap();
+        self.ctx
+            .set_config(Config::Configured, Some("1"))
+            .await
+            .unwrap();
+    }
 }
 
 /// Load a pre-generated keypair for alice@example.com from disk.
@@ -60,20 +93,6 @@ pub(crate) fn alice_keypair() -> key::KeyPair {
     }
 }
 
-/// Creates Alice with a pre-generated keypair.
-///
-/// Returns the address of the keypair created (alice@example.com).
-pub(crate) async fn configure_alice_keypair(ctx: &Context) -> String {
-    let keypair = alice_keypair();
-    ctx.set_config(Config::ConfiguredAddr, Some(&keypair.addr.to_string()))
-        .await
-        .unwrap();
-    key::store_self_keypair(&ctx, &keypair, key::KeyPairUse::Default)
-        .await
-        .expect("Failed to save Alice's key");
-    keypair.addr.to_string()
-}
-
 /// Load a pre-generated keypair for bob@example.net from disk.
 ///
 /// Like [alice_keypair] but a different key and identity.

test-data/message/gmail_ndn.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a1c:b4d7:0:0:0:0:0 with SMTP id d206csp3026053wmf;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Received: by 2002:a5d:4651:: with SMTP id j17mr19532177wrs.50.1589819005555;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1589819005; cv=none;
+        d=google.com; s=arc-20160816;
+        b=IZbNnzzuYzTFuqfuZwpd3ehqpYYGpn31c8DsfGbQ8rpbS0OTTROkVYvihQl8Ne/8X/
+         brEWsrcmaCh9WpFMzpI+cp/TY39uusnI6qdp5rcgrFmFgoANtwf3TBBj1+f7wBPn46BP
+         dQOUsg/J8KVfvzVgvL1x4uyJ0m9QirDgJeJ/BvrswbTleRQK7oY3fIireUCDxj6r2lCB
+         1Z0TKw1mgIb1LiFMZz8kvCNn3R4KSFnwS8rIju0hYwnsioNiExVQgumXL+RVkEZ9BMzf
+         UdoWIAw3VW+MOZFTpfLCEfgIPtLg/gtE0Q1P+a3KKpi8dkPiV2n6DGMecy9lTLtdhCXt
+         pnaA==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=nNP0DktrSjdBaFfhhoDi2O9KVKM0iXE5ZgubQ0q0ff68Z6Ke7c8dDBXEsZoToI0s4Y
+         w90KyJFpgMJLFmP3iVDRqCfohi2y1HGdWg5VXQPTvzM7+YozZRlbNNV9UsuyRY91CXrJ
+         a2XREBgB+LPMGQivwcHtUMZfyNv/4uiwWivk+92ySNDhxqOiDt4R5Jak/7RkZMFwQpsE
+         JGwk6asM6VqZlihkF24lKv3pPaob6feyX3wD5N0+Mqiy1kQTj2JkpQk6nkTmdf0gapZe
+         fOhU1NkbNfbuS3U7m2gEUiyktE+MhV/MgAzgBhm9bgNt2gQLVWju8rHkPndfv1PDmEkC
+         FsYQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id s18sor5584435wrb.25.2020.05.18.09.23.25
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=dPisws+OwGFyOy0a612XYZgvz5T71GcJRJtU068/Tce8vN/+ggIQtUsZnZtsphe71v
+         2NvfP9ULxR4cXvomTvhrYAk19KdxN/S7SeyBbmXv3x/tg+DBVCmmPS/6RXrcl6Ms3Hkw
+         uPFQ9S3KcvHe/2bcb5LSTA/stIP4tuxxAXvsX2j+MjPYPWKAl50jkSbWK98U0Q0U+MTl
+         pKaaC9s9iEBafac8BFZCy4DfpumKlemNEyRa3cSV2hw+DYHKA5peModrK1A2tcsfstFF
+         rZi8yF/D90RIFbE04DI2QCxB3trsChNF1aYF06aSzI//wsfM1+lb+uGPi0YVkw3n4HrX
+         Xw4w==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=A/NCOtgbpA7VzB1G7ZFo8TA2FfrjuqjGdwMrJr3yXe21FrBFwzssprJwOkynqoVLkK
+         iJU7uMF/KTcQPDEmOLFThzFfe5GCx7eJtZPhwY+FbBlC5sq4I55/xaQLd0gOZ1BYXwMn
+         2bk169d2aoukbaLbGSQZF3d9atd+/e48YzkRxpmUoLcrWk2LcHAeQIG7SgT9pfX5DKPr
+         VpxM5/GMVEBbTRhBIWCeVSfpYCs80l0xEeTC3/B5lzpzMVDE8QCW6Dwh75b4Tb2K6yru
+         Zsy5ZpRmwv0wrkrb2vM+pl4IMkaF7s8XosIvlIT++fQV5xDFItT4atpykZvSDB92RKV0
+         8lEA==
+X-Gm-Message-State: AOAM532RG/PT3ChZHBCDORGLtAjKvX8TGBuOy+AxrnEaJT6v1ieb+VV1
+	+ejly+/6UthxHYlkOJYAszCSgL4dKVFotoVaN7LhEA==
+X-Google-Smtp-Source: ABdhPJz6veVKWhomCL4gK+whrybuMzHCDCq8AowgQvi7sobpMoM/k9CDw79jo1j3OUcTz6MEeUYLxEXuNIuu4zyoS7kVtsUYryGFHAI=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545185wrv.159.1589819005394;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000012d63005a5ee9520"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a5d:5183:: with SMTP id k3mr13704211wrv.159; Mon, 18 May
+ 2020 09:23:25 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ec2b67d.1c69fb81.213af.67a5.GMR@mx.google.com>
+Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+In-Reply-To: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000012d63005a5ee9520
+Content-Type: multipart/related; boundary="00000000000012dc0005a5ee952f"
+
+--00000000000012dc0005a5ee952f
+Content-Type: multipart/alternative; boundary="00000000000012dc0705a5ee9530"
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--00000000000012dc0705a5ee9530--
+--00000000000012dc0005a5ee952f
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--00000000000012dc0005a5ee952f--
+--00000000000012d63005a5ee9520
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Original-Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+Last-Attempt-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+
+--00000000000012d63005a5ee9520
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=a185ogBcMzF9whNVWvuyUoUunNZk3Vc1kEIFmPkX0IxLpAFcI+fOQajOSromGl7Oyi
+         yecLwQevpww2Xd0XjZ3UkZvrI9m9koRmh0QeoHvgTRORiVwj08+PVc3N4F9bCO4w9i0J
+         ir7SSsJqBCDovoIFSFDyNa64vs6Nxno0cH/DaPG7pVTdD+3jfB7nLXIsMQYeX+1eP6rB
+         UhKxH82r7Mh9CI2PWDQpVtGj63AMUEyHgE9Ou08PWbbKjrQOasoG3Tw8tB1GoN1XYssM
+         rxOTgWEoTiduZ35AUH6h+eChOn9OHuI3SPECcVob70Qndayia3dMKfHMO6sEx9J0Wpic
+         29vg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=miGIfL5BgnkD3wQvS34RtGwRRoh+8gJT5sFFfdX/hVyG/dvjXfdwP4yyNWr8ox8iY2
+         BLlahS4y4VGcbG1e2aYjurnWNytGu6utQcZax/uUngJ0bTOwXW1VaIiEZtqd6gTV+8d/
+         rrfQ459+4vXqIoQf0+Oi/U6dWwgJvPPjjRiToWdF3vIJE8R1iTRdZbW4lkgxSADbmskg
+         noT/gWGWblHtR6uuGuKGJ3bkhJKCBnjavKh0LlbWEeFBZfmVNPRvzEFWHjBDdu5wvSL5
+         0QJ+Qn0Orfn5CJuN3xPfzT1S2rI2iYZx37KX9zyMnZEx0ilkTYqCtBPWkrXRYDSXcxYS
+         Y1ag==
+X-Gm-Message-State: AOAM531vhwpXiK8M12286dOJx0Q5fBl9ZaH6BJKts93GoxvPv0xdryP0
+	jg9wYmoP5MUHudsxAMCYDFsCUMVx2PEywyIsaQqklw==
+X-Google-Smtp-Source: ABdhPJxlVJtTODM3pZZSTbbpAAAQRU8XbmuosDF9fgQZmVwxGZSzRWl22o+moppVRU/r8xMAyf0r3+qXwEBe1vZfjZo=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545162wrv.159.1589819005034;
+ Mon, 18 May 2020 09:23:25 -0700 (PDT)
+MIME-Version: 1.0
+From: <alice@gmail.com>
+Date: Mon, 18 May 2020 18:23:39 +0200
+Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+Subject: Kommt sowieso nicht an
+To: assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000000d652a05a5ee95df"
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/plain; charset="UTF-8"
+
+Wollte nur was testen
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">Wollte nur was testen<br></div>
+
+--0000000000000d652a05a5ee95df--
+
+--00000000000012d63005a5ee9520--

test-data/message/gmail_ndn_group.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a02:6629:0:0:0:0:0 with SMTP id k41csp368502jac;
+        Wed, 10 Jun 2020 05:17:57 -0700 (PDT)
+X-Received: by 2002:a6b:1448:: with SMTP id 69mr2898530iou.83.1591791475733;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1591791475; cv=none;
+        d=google.com; s=arc-20160816;
+        b=a0vSKJPbMtGYFnuk1ye/gnnV00Zvva4OOJTMOyfm13xMJD0YAhzGVfa7Z+wn5sQ8dw
+         VAxpmDHCkjp4jol0C1iutiq2Nl0qma819oFPuuoMLLatKQXHpo8Jt+sL3MnwNR7J5bZC
+         1c6Fjk75EIsRWhJd1HCkm44A6UYHxqqsTnzQCaNiHbjsRsvbggxwlMGSrZ4silxqSDvo
+         Pzd/YDLCvsnZNSNIjIckKAwtGmY6sXctZ+JnOTykXAyL32Milfwy1vRL9xm10Q14biTR
+         +qaIQp4E6WE63g1WHvfAjs0Dru7DalUh4GGl/NAwqVhY1gVyRD5E9/nODyHAfxjvaxDD
+         4sMw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=OJbgbrktMKyczw25z/ib7lSdRX80PEK3Myh9fj4q6mDlXmPPv//Gv069znRQ4QbadM
+         HUXZH0WLMZcGyqI6SvGL/noxQ1O8yP0FYJJKTkoX0Gk2hHzfaE3x1scOP/o2FMMQXIFm
+         S4CgGBD6HHzBJYj/rSL3gzqLzx1Id/z5kTeDvH2cn8JJAcCE2q/nhjTyWUb87geoNlDJ
+         A1HRrLHK/0JOyRjHfg2zZCqIvSi1xmpiHStMyL9mfVyrQs98tsPxaOkJHjLplFARoPlr
+         mmmDvsFg7MPvFqkkANzz4JDHidnfKRULCgnrVj1yTU66UagUpQEGjZqz8/99YuU6nt1t
+         81sQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id w14sor16686480iow.23.2020.06.10.05.17.55
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=aO4aNy7CUOk9O4Jnsue/DvMFY6Ph0C34AbpoxJH+mLZpOmt/KYGCGYWgunZgamF15U
+         Vm8JY5yLKGwkTz2m3abDnKNP4fpl6zeZ5fyk5LvXH2Jema0iocHai6pJZBoFGPnonNmd
+         MscTf1sEltbOxwfOmM1BRHX34c1jW0+8Yd2+Nhg2DPvzuq1brOVin6bUV4VX5EeeuNqT
+         ZTewjJVPmO/B5NQhdpG81FO5w4hKSQ/VzZXnap2thMf3gOmnaoR+tbsnOIAiklcLdJ7b
+         57SKUwI041pwSmh9dffs0STl2GvMRSJyGCtBqMnzXgflqoGTcnPflWgR3LXHM/MIA0q8
+         WqRQ==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=iORAzNvXegQ8oSp4RYb/S168muAiBox769seMk49kDBIvXwI+N8P4mUZq/zDi8DmQd
+         +wlLzVzowQq6EofiSpjOJWT9IC/k8otk15PMGtgHE4BGSSeKn7L30d3ocQS93HzYnLmA
+         VBlHBdFTKrsfKhe2+CQyCosTDGRpbkQLuRRyhxChEq8ltvaOHgbu1+eCeb9PsPuh6OxH
+         kvTHJZeA9A+eLOl26pBmqGIWkr7FlYW0wI6YPoEs9WXX5LSFOQs6fm/9l366eIR7IFFI
+         ihX5LrZl/Cf0lwwYX7fqIMgnHy1K+QnKuEb+dRQGqLbxdIEls9bXIF98iPQVkEWzgSZy
+         ip8Q==
+X-Gm-Message-State: AOAM531ahfHE6oS9/nuni8pNf9bwC+DXAcaLV0owBwNCj9kcTPLCCNhX
+	W1JNciK0ivEIVB4dgiyLE/5K7iKbEznQhqyG9Bi1QA==
+X-Google-Smtp-Source: ABdhPJygljUXswH0ycJyHmXVthi5IjlDvP8QdYlMdHUPKEtgIZeUk69Acti5LnswGhg63T9/L0PuGZGBM5XE5BsP0mMNNDRZyt+DgnE=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990163jab.84.1591791475516;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000074432a05a7b9d512"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a05:6638:101c:: with SMTP id r28mr3059870jab.84; Wed, 10 Jun
+ 2020 05:17:55 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ee0cf73.1c69fb81.6888.c2f4.GMR@mx.google.com>
+Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+In-Reply-To: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000074432a05a7b9d512
+Content-Type: multipart/related; boundary="000000000000745e0805a7b9d51b"
+
+--000000000000745e0805a7b9d51b
+Content-Type: multipart/alternative; boundary="000000000000745e1705a7b9d51c"
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--000000000000745e1705a7b9d51c--
+--000000000000745e0805a7b9d51b
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--000000000000745e0805a7b9d51b--
+--00000000000074432a05a7b9d512
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+X-Original-Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+Last-Attempt-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+
+--00000000000074432a05a7b9d512
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=eyr60XbgOrgHoZFpRYzw9WQIR7aEBaYKWhiEcqdnugB+hn0W2KVcTkKiL2C6zSF+jh
+         l+lM+dNZZTUcMqWx4kVgTVtqwUNea8OUqe+WLqx04ULwdKZn1okbKYovaiavCLKOKDnf
+         ZP5mNz3Ka/ywpCGoq8rdgnXc7NunnkWeaBpYY/BWOmLU4WNXX8zS7etXXhQE4YPQEJT4
+         Sh2o/YIIjDLncJFMyE+25n3tbd2mIoLt4sjaCHE5ibm9w7zojyHM+LDCQ37cM74FEAAa
+         88KTn0gSnCFBCfojhfxOH78CpySHG3FFfTlpCefwP2A5J9MQlb6QdSVa9STYSx3IntJ4
+         L7Tg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=pBL4/bKUDw5E2zo1uR2Tl69h2iTlMgIAcnzQgodPCbU4jZ9kH+F5H9rfbzXCjT06J7
+         L72SYpdfgc5fOwM4GhRcdYnyK3wiXQ8ugpL19nbYt2iWo/vRF3GidawXXDGb2GUYpkzX
+         1Mz531cy2/HOsmQbUQ7304KV+OUghtcg8eLNnFuhQch7n12Kk3yy3AOzjrLoktcdgIsy
+         /HxBjyut0Au+A2t6si+PVwTHvC647a0BioeV0tUYLigzu3/jgP9Hb8eRZaXTX5VC6iZi
+         9QMH/+rXp05IK7OpGWh22xDpeV8CDkQ2sLFaBhKxtJ+nYoerM64t8EJXBBsVQb18ojGz
+         pW/A==
+X-Gm-Message-State: AOAM5330q6kn/TKataMNEVigNfNdr/xii/PQgHXzJyMbwLvsETlNfLoy
+	1rM9JBIGrcHeEDRx4qhZfl5S4bircceU7c3i6Fyn2fRO
+X-Google-Smtp-Source: ABdhPJwysG+S90b/g+9mK7LgeHhmJTBowst6JMhL16+a0coTi7P1NVp9jjaNHJfhvhLodYG6eHIvWdbQGJnAP2brEzI=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990137jab.84.1591791475066;
+ Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+MIME-Version: 1.0
+From: Deltachat Test <alice@gmail.com>
+Date: Wed, 10 Jun 2020 14:18:26 +0200
+Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+Subject: test
+To: bob@example.org, assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000006d8d7d05a7b9d5b3"
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/plain; charset="UTF-8"
+
+test
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">test<br></div>
+
+--0000000000006d8d7d05a7b9d5b3--
+
+--00000000000074432a05a7b9d512--

test-data/message/gmx_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Received: from mout-bounce.gmx.net ([212.227.15.44]) by mx-ha.gmx.net
+ (mxgmx101 [212.227.17.5]) with ESMTPS (Nemesis) id 1Mr97m-1jC6Y01o86-00oEqk
+ for <alice@gmx.de>; Tue, 09 Jun 2020 14:35:30 +0200
+Received: from localhost by mout-bounce.gmx.net id 0LhiZF-1jDTj11ZoH-00msO3
+    Tue, 09 Jun 2020 14:35:30 +0200
+Date: Tue, 09 Jun 2020 14:35:30 +0200
+From: "GMX Mailer Daemon" <mailer-daemon@gmx.de>
+To: alice@gmx.de
+Subject: Mail delivery failed: returning message to sender
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+X-UI-Out-Filterresults: unknown:0;V03:K0:O8yx6kuPaGQ=:0wIDPNXEr0wX2oNsLnXaWA
+ ==
+Envelope-To: <alice@gmx.de>
+X-GMX-Antispam: 0 (Mail was not recognized as spam); Detail=V3;
+X-Spam-Flag: NO
+X-UI-Filterresults: notjunk:1;V03:K0:QcE43EBhMmU=:IC5vvzi9jhPS/698Wuubzw1Q4N
+ h87X9j9B3CBN0ZKXB67KepwyNHmh9pxmFIUOMimylv7UK9np+j3X55roOd0nX9BmaaZ3Twvqf
+ UaSsxmyU+cNr6m3+oOb3udJBLe2pJEZDk1cOwACb5NXzYPSaIj4APfGCyvrzIx3FGkNuScNBb
+ tCbbKUJ0GB/VmJLB34XfF6dNN+Iwv9IQ9Yrvw/VXv9vWKsi3qRGGUt3yRw5jUKhQlBY21Pnoq
+ m0LqoMbAKfH1tKEQ/5TymH1ei50YKyWzZ89ISkQwkbYLaqN+6meGACpY18j43VCU9Fk4WQR7y
+ 3XvBYh2CO0CnCn+M9VsnasYag2sNrySe9nzyKfRTaxEg8qlJtl7kS4GX/FsxhHPavkqnU62Gl
+ 9V5TxIG7tmIR0Bf11sPzG/WGegoOHxrfz+qYR81llLMOHznpdDRKjsYDtO/rFBGZzYTiCZsrW
+ dZPVXV25SVcrDGZOaop3JoCbglmXLcSLLhmfE5MzyJEGte3I+6EiZJNeIe8qN3wMDTsRtJL9S
+ J6b2F/5/kTGVAWnXtNlf69BholCrxvjC4Snt3Xjc+7WIO8iw2c5YjmWy+4bAwd4uWll529hZd
+ 6pUYGwjFRnKleivCaJIt7DqbvbE7GZSbQH8fXm3zYqYTrrxiWdlykdoOA1OGbeM06RHJt3mJB
+ osZPU8BZKt0OiBOW64vg6gyAsNC0f02EA7dvRWYgFYqlSogfWZQIOKDKibMVHpIaA0foXg4BG
+ TEQDlsTIL0n2WC9WVqkMdm6xUXHgpArCrAsUhw3mEqPywEfJeBHn60tP2vQ9+pDIQAj5dQCDV
+ y96qSiCX4p31HfrWwAXB9mHfl4OO/tPcKUGBclj2rZ/NMc4O+7yDedLWXQnRzQExfOJLBbBh3
+ xgiNlWFHvDLn0pKG9EI1+3wJ7m2GF2jzDtbQTBv9z26DuAq5WbHZHupzeyfP7VCVXcKuB6sG1
+ 3+LWcdYtcXfqT58HwcvDLwowC4uJpiHfHwtVdiGMtHnmYLysp0V425g+vofQfNzBgR3d9JC15
+ G+HS44o6x6Legm6KnHYH3k0KhR7fgcgswJv/S+I/ryppUhGb2jezVZIUzgvAplzIUDAWnrHdF
+ KVqZ5wBJ0acShIfgMlsIxnBmcnIQ4R4jq3zAyj4XTFxVUFanU8ySiXubxV5PzJqj+GsVa2sjx
+ 9n/xQRJLwgMC4BYqzP6lEPwg/g5AneDAnl7ZlcQPC4SCMblC8N1KZyyIDTXPOI/o4lfdMYb9P
+ 7DmBp2S8aA2yuDe5XT20OmX3kVWeBOsBaAGvVFpIn7gwIDqnFh9WSMP6mkCwfChN3D1yLquYB
+ KAODgRZV5lVNmK+eOjW8m2oiRxmfxrjXLtw5PEhn3RkiRN4HnoePJeoYC7SG4EUwg+wYPu3M6
+ exP/YigoE6bjuBS5d0imUTRDMiwg469GyrFo1J1GkRVvj3lXSF4Nt11j6waqu1l0ReDYfU+QM
+ EMPGLEh1vRChCaqz4L7YI5FlSAVXxfmst0JRyE4k5r9CToEbZuYlPQ+jbcvptwBSaqzMb/YfT
+ cCrU2RWHUfmYIy8x8A/t8ScRbYPzs1lTK3yn1hYeXpw8Fgkip6DIIXAJwUUp+2SLcELICIo+p
+ uumMN0P/OZHH3V/hZ0dPr9xsYi7gdd/vyRIRPUwiL1rSp2WJGi+w4atun8kQBgnbAznRObDh1
+ 4zzKhApX9jo5gtFN6640QDEI5KpsMuPoty4rj9OK163ntKWGR451n+5ZX2FilTlpZYlIPO9Hy
+ SrjHzBog4texceR1OLh4pb/WFB0XFSjQchPAltXCYQFs82aDDk/A0nOPk=
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of
+its recipients. This is a permanent error. The following address(es)
+failed:
+
+snaerituhaeirns@gmail.com:
+SMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please
+ try
+550-5.1.1 double-checking the recipient's email address for typos or
+550-5.1.1 unnecessary spaces. Learn more at
+550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21
+9 - gsmtp
+
+
+
+--- The header of the original message is following. ---
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=gmx.net;
+	s=badeba3b8450; t=1591706130;
+	bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=X-UI-Sender-Class:To:From:Subject:Date;
+	b=NwY6W33mI1bAq6lpr6kbY+LD2hO9cDJBItTgY3NRIT94A6rKTVlSmhFM3AxYgFnj0
+	 Db0hncsNRDqcdtRoOo8Emcah5NJURvEQohG37lkug3GqneB4+FNTdYCeQbOKlZn6on
+	 pYYD/T9CmeL2HG3+8voeBjZIUenyXrF2WXG37hFY=
+X-UI-Sender-Class: 01bb95c1-4bf8-414a-932a-4f6e2808ef9c
+Received: from [192.168.178.30] ([84.57.126.154]) by mail.gmx.com (mrgmx005
+ [212.227.17.190]) with ESMTPSA (Nemesis) id 1MKbkM-1jNoq60HKm-00KyL2 for
+ <snaerituhaeirns@gmail.com>; Tue, 09 Jun 2020 14:35:30 +0200
+To: snaerituhaeirns@gmail.com
+From: Alice <alice@gmx.de>
+Subject: test
+Message-ID: <9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de>
+Date: Tue, 9 Jun 2020 14:36:10 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+X-Provags-ID: V03:K1:7awoptmynaF5MqxCAincdsa7zFCwQFNkFb6xRxigK06uEiGN00b
+ Mv2wJU91CEd4mvCCWzrTtaWLZDLH8pjAWaT4+HvYIUbpwNx2jC6WHTppkYYRMVJnm4lG9pr
+ SUx1OlIcp0kbsnl3mB9xYNFwm9jzpR9Kx8QEHwIbZiiSFBcH56498UGQi//kKXVMos8C14o
+ I7cmwYmr8xB09DwLMKXfg==
+X-Spam-Flag: NO
+X-UI-Out-Filterresults: notjunk:1;V03:K0:NWQAUbhAkBc=:yAaolOnVCDWEZhgUwwvtEs
+ wXbSJ/GfMvDRpCkYpFBvHXOTpGm6hjdjQ0vLK2hvu/Hz22UdlWbIdc1J2oO9S5U20mIdc+1bS
+ TPSSpqPFc7ICPx4Wbvv2SEp9ZqH2q7ORC52UvUWfI6OjAJEPDNrXQFdUiZAa72hLj1NPeG6Qi
+ 4AbL2HwLfJ8s6TeOCm6TXRRuD+w1o/ASFOqQmoao2dFyZ2BaoAgOKPKxXYfwVGceuUygpchyS
+ 0d2bZYOXSLR+6rUYevjZAq1OCi9AIC6/wlkOe5yIRk4gJFMfPauaICsdnq3uZ9ikCAX83VWun
+ PJVMxTLTP54lgo2h0jMBX3uKk10+/wzXWplllxX9NnSa3x1V28n6raslNF0IoC6Pm72kC6Jzr
+ GkC22viCm3/Y4uHlPMOXbY5WFrQe/D9GKeJeXBLoGciNwIFkUG12a+iqWtoT+h5HVObTW8LxM
+ +UtEl97nAwxYSM+sGfIpasRpZc7r/SgN3JWGO9R9WaXpW4Cc1dH7RI+hzuZDsDUBEGTUTVPDo
+ 0SvjKHiJ6sUqGDyfv4HUgVutus6EYP27LALND4ekfom2DPRFopZhbtV5fZT7CL1Q1NogU7tYf
+ /FdmR1T1J1zCAZSFvyR5LBkfglZlHzgdnTF2heuxyqKq4dm0hnLFSULB1+CVWsg8hzrruvO5q
+ XzA6qIhBQUZmWo7wBpqpkBPxzjgTGGtXc5y5e6+crxYbbuQdnUWEnyw2xI4d6pJPqtHDA2/vT
+ ZgvNDUGceavTR5Rtyb14hhX4Q6dWK3ATy16j4hs9Aq+q/IKyVAX3A5nFYbJRIz+2YnoLr2YOa
+ IrScEorXjvTxjw+aBy73SZBe2REPzJ+O6k7chVrYjV9Q28FiGVuRYJYxWw/59Pes7IAbmfQBV
+ 4vqGCQQr4eG78gVwjw+SNdp4/6jdNkIHDqR4XW9id/r5wYxKKj4UUkSor3/+h9Rd9srh+GApy
+ uOxw/ejFvbRcxFIjvadpq1KLnO7nM27nJ4lp44ul3i7VUGefLM/45TCsuds2HM1iQWhPFQ54y
+ SA5sYjf73EUJdkHchaf5i+4uSOmbOWQ4Yvmd8+IoyoXAxvEzY2Xh53nWi8ZPY1Tu4Bw8GRrz7
+ L+VK0QiWCg3/hM7wRlFFyshmMrScMk5fOf9ynqd0JbHB7u+n4/GUwx3im/w8+NgSd3YOz7wNU
+ KD1snDWoMUO8f23Ik1Osym688OLWNwKYT+mZbMIMXcz1fB+olRZn4czMhN5DiSb8hyOxRI8NE
+ PNfaoN87CXiRkgazV6U1eiRkfcK2AvI7zOJF1tclUHZ9awyYoXtxfEzZ+J/2TCXiC7V2iSkUF
+ EjwgPxlJmccccjsxc46v1ajnTxLo0tJbZ0+DJXWkCgQ0d/iiScQ=
+
+

test-data/message/posteo_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Delivered-To: alice@posteo.org
+Received: from proxy02.posteo.name ([127.0.0.1])
+	by dovecot03.posteo.local (Dovecot) with LMTP id zvCFJRzX317LGQIA+3EWog
+	for <alice@posteo.org>; Tue, 09 Jun 2020 20:44:24 +0200
+Received: from proxy02.posteo.de ([127.0.0.1])
+	by proxy02.posteo.name (Dovecot) with LMTP id mhNkNAnR316xBQMAGFAyLg
+	; Tue, 09 Jun 2020 20:44:23 +0200
+Received: from mailin06.posteo.de (unknown [10.0.1.6])
+	by proxy02.posteo.de (Postfix) with ESMTPS id 49hJtv3RRcz11m7
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+Received: from mx04.posteo.de (mailin06.posteo.de [127.0.0.1])
+	by mailin06.posteo.de (Postfix) with ESMTPS id 6935920DD2
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+X-Virus-Scanned: amavisd-new at posteo.de
+X-Spam-Flag: NO
+X-Spam-Score: -1
+X-Spam-Level: 
+X-Spam-Status: No, score=-1 tagged_above=-1000 required=8
+	tests=[ALL_TRUSTED=-1] autolearn=disabled
+Received: from mout01.posteo.de (mout01.posteo.de [185.67.36.65])
+	by mx04.posteo.de (Postfix) with ESMTPS id 49hJtv001Vz10kT
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Authentication-Results: mx04.posteo.de; dmarc=none (p=none dis=none) header.from=mout01.posteo.de
+Received: by mout01.posteo.de (Postfix)
+	id DCB6B1200DD; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+From: MAILER-DAEMON@mout01.posteo.de (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@posteo.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="B39111200B9.1591728262/mout01.posteo.de"
+Content-Transfer-Encoding: 7bit
+Message-Id: <20200609184422.DCB6B1200DD@mout01.posteo.de>
+
+This is a MIME-encapsulated message.
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Notification
+Content-Type: text/plain; charset=us-ascii
+
+This is the mail system at host mout01.posteo.de.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550
+    Requested action not taken: mailbox unavailable (in reply to RCPT TO
+    command)
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mout01.posteo.de
+X-Postfix-Queue-ID: B39111200B9
+X-Postfix-Sender: rfc822; alice@posteo.org
+Arrival-Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+
+Final-Recipient: rfc822; hanerthaertidiuea@gmx.de
+Original-Recipient: rfc822;hanerthaertidiuea@gmx.de
+Action: failed
+Status: 5.0.0
+Remote-MTA: dns; mx01.emig.gmx.net
+Diagnostic-Code: smtp; 550 Requested action not taken: mailbox unavailable
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Undelivered Message Headers
+Content-Type: text/rfc822-headers
+
+Return-Path: <alice@posteo.org>
+Received: from mout01.posteo.de (unknown [10.0.0.65])
+	by mout01.posteo.de (Postfix) with ESMTPS id B39111200B9
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Received: from submission-encrypt01.posteo.de (unknown [10.0.0.75])
+	by mout01.posteo.de (Postfix) with ESMTPS id 8A684160060
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=posteo.de; s=2017;
+	t=1591728262; bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=To:From:Subject:Date:From;
+	b=brJnt4PLAX3Tda1RHCo91aB1kMAL/Ku9dmO7D2DD41Zu5ShNsyqqyDkyxb1DsDn3O
+	 6KuBZe3/8gemBuCJ/mxzwd9v8sBnlrV+5afIk0Ye9VvthZsc4HoG79+FiVOi9F38o0
+	 DtJJFYFw/X7mAc5Xyt0B0JvtiTPpBdRAkluUQm+QW6cW6GGlwicVW19qvebzq+sHyP
+	 X2bZ8wpo78yVgvjPBK3DLaXa+pKFMBjLdDUcIE2bZnY6u6F1x8SXGKGBoxVwdJipJx
+	 v14so5IejNsf4LYJjH3Qb8xgK1aAi6e6nQn4YXV0INL6ahzgALiT9N6vwunNKYVJNi
+	 fPPKvBWDfUS4Q==
+Received: from customer (localhost [127.0.0.1])
+	by submission (posteo.de) with ESMTPSA id 49hJtt1WPbz6tmV
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+To: hanerthaertidiuea@gmx.de
+From: deltachat <alice@posteo.org>
+Subject: test
+Message-ID: <04422840-f884-3e37-5778-8192fe22d8e1@posteo.de>
+Date: Tue, 9 Jun 2020 20:45:02 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+Posteo-User: alice@posteo.org
+Posteo-Dkim: ok
+
+--B39111200B9.1591728262/mout01.posteo.de--

test-data/message/testrun_ndn.eml

@@ -0,0 +1,107 @@
+Return-Path: <>
+Delivered-To: alice@testrun.org
+Received: from hq5.merlinux.eu
+	by hq5.merlinux.eu (Dovecot) with LMTP id Ye02K6PB5F43cQAAPzvFDg
+	for <alice@testrun.org>; Sat, 13 Jun 2020 14:08:03 +0200
+Received: by hq5.merlinux.eu (Postfix)
+	id 9EBE627A0B2E; Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+Date: Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+From: MAILER-DAEMON@hq5.merlinux.eu (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@testrun.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="CDB8D27A0B2C.1592050083/hq5.merlinux.eu"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20200613120803.9EBE627A0B2E@hq5.merlinux.eu>
+
+This is a MIME-encapsulated message.
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host hq5.merlinux.eu.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1
+    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in
+    virtual mailbox table (in reply to RCPT TO command)
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Delivery report
+Content-Type: message/global-delivery-status
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; hq5.merlinux.eu
+X-Postfix-Queue-ID: CDB8D27A0B2C
+X-Postfix-Sender: rfc822; alice@testrun.org
+Arrival-Date: Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+
+Final-Recipient: rfc822; hcksocnsofoejx@five.chat
+Original-Recipient: rfc822;hcksocnsofoejx@five.chat
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; mail.five.chat
+Diagnostic-Code: smtp; 550 5.1.1 <hcksocnsofoejx@five.chat>: Recipient address
+    rejected: User unknown in virtual mailbox table
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Undelivered Message
+Content-Type: message/global
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <alice@testrun.org>
+Received: from localhost (p200300edb723070079835ce22985a199.dip0.t-ipconnect.de [IPv6:2003:ed:b723:700:7983:5ce2:2985:a199])
+	by hq5.merlinux.eu (Postfix) with UTF8SMTPSA id CDB8D27A0B2C;
+	Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org;
+	s=testrun; t=1592050082;
+	bh=Kvhta0OMsTRVC7OlaAqo68TBE0KuGBv4vUBp6Ez/7VY=;
+	h=Subject:References:In-Reply-To:Date:To:From:From;
+	b=Ql60JEGFXLNvjsyihATw2z34ct++8xZvTPNw0snXe6+oqdqsRZJ9tWNDTxOgx8Iqf
+	 HQ4puBVGcWjIlszYQVLlq3APi04o2ep3GrD8EF0J0GpDdW8yw6wCos6Q8r+TWmXwET
+	 kGXHTRPVaUIqZF2i/utypxMfd1ua0S3jBDnIXTe/p2+XvfC3Cf3hZGW+FQ/Zd7G8Vh
+	 /U2rgX5BTIGf26ZCbmcMaXWkftgv6+yns0AmzorV9yB+EhTkWIUjk+C25bRtMbJ5mZ
+	 93dwdr+sXrrSZLSi+LBqc57Dv9j4p/SUmB4zPlvfUv7/bqLi36pypvtCJ5Ul8UEXSb
+	 XNFZPaEl+mwjA==
+Content-Type: text/plain; charset=utf-8
+Chat-Disposition-Notification-To: alice@testrun.org
+Subject: =?utf-8?q?Message_from_hocuri1=40testrun=2Eorg?=
+MIME-Version: 1.0
+References: <Mr.VSg3KXFUOTG.9sn7JBxZn1W@testrun.org>
+	<Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+In-Reply-To: <Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+Date: Sat, 13 Jun 2020 12:08:01 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@testrun.org; prefer-encrypt=mutual;
+	keydata=xjMEXt3z1xYJKwYBBAHaRw8BAQdAf6MctU/8cmEqwEN9VFZ3gHBFIxKiEaARZl1DFUkI7e
+	rNFTxob2N1cmkxQHRlc3RydW4ub3JnPsKLBBAWCAAzAhkBBQJe3fPXAhsDBAsJCAcGFQgJCgsCAxYC
+	ARYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygAG2IA/1nTmmmkHAc1Bjtx2FOstbaS+N
+	XHjxaK+hkoWllsyhz0AQDJJ1++u7jVZPRn/j1LlByrT3Jv/D1aY14J5rjj+ADVBM44BF7d89cSCisG
+	AQQBl1UBBQEBB0DpSTaZ30dAVwM9PkBe2h+gFyxn9HSorP4XCHJu/lIdPAMBCAfCeAQYFggAIAUCXt
+	3z1wIbDBYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygA2QkA/16toWCtseYKw8G1X2j7
+	xYR3Cyabq37hgbesDOThIIzNAP0UCUS8mnunmkS5adEbftRaDi2JZoGxDw46jtJJ2+13Cw==
+Message-ID: <Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org>
+To: <hcksocnsofoejx@five.chat>
+From:  <alice@testrun.org>
+
+F
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu--

test-data/message/tiscali_ndn.eml

@@ -0,0 +1,91 @@
+Return-Path: <>
+Delivered-To: alice@tiscali.it
+Received: from director-5.mail.tiscali.sys ([10.39.80.174])
+	by dovecot-08.mail.tiscali.sys with LMTP id SBRfEpGb517VAgAAd2fHbg
+	for <alice@tiscali.it>; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from cmgw-4.mail.tiscali.it ([10.39.80.174])
+	by director-5.mail.tiscali.sys with LMTP id MFUPL5Cb516tawAArQJVuQ
+	; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from michael.mail.tiscali.it ([213.205.33.246])
+	by cmgw-4.mail.tiscali.it with 
+	id rTtS2200V5JdeUd01U2RlV; Mon, 15 Jun 2020 16:02:25 +0000
+x-cnfs-analysis: v=2.3 cv=ZdPMyfdA c=1 sm=1 tr=0 cx=a_idp_d
+ a=AfTPebshMYb+aQOCLa9q3Q==:117 a=HpEJnUlJZJkA:10 a=jmdcTMp_Gj4A:10
+ a=r77TgQKjGQsHNAKrUKIA:9 a=b8iBRs35AAAA:8 a=NMdB-582e605uxHDr_AA:9
+ a=QEXdDO2ut3YA:10 a=MhhPCb74-dYA:10 a=HXlsH_Kov2KnitTn7A4A:9
+ a=BkuCPOF3BOzethIN9HQA:9 a=qG5HpJ6ZyD35YNEB:21 a=kvHihYffoorsyJbA:21
+ a=xD8EQi6zkreDqSNPYj5l:22
+Date: Mon, 15 Jun 2020 16:02:25 +0000
+From: Mail Delivery System <mail-daemon@smtp.tiscali.it>
+To: alice@tiscali.it
+Subject: Delivery status notification
+MIME-Version: 1.0
+Content-Type: multipart/report; boundary="------------I305M09060309060P_896715922369450"
+
+This is a multi-part message in MIME format.
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/plain; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+      This is an automatically generated Delivery Status Notification.      
+
+Delivery to the following recipients was aborted after 2 second(s):
+
+  * shenauithz@testrun.org
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: message/delivery-status; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; michael.mail.tiscali.it [213.205.33.13]
+Received-From-MTA: dns; localhost [146.241.100.150]
+Arrival-Date: Mon, 15 Jun 2020 16:02:23 +0000
+
+
+Final-recipient: rfc822; shenauithz@testrun.org
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550 5.1.1 <shenauithz@testrun.org>: Recipient address rejected: User unknown in virtual mailbox table
+Last-attempt-Date: Mon, 15 Jun 2020 16:02:25 +0000
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/rfc822-headers; Content-Transfer-Encoding: 8bit
+Content-Disposition: attachment
+
+x-auth-user: alice@tiscali.it
+Chat-Disposition-Notification-To: alice@tiscali.it
+Chat-User-Avatar: avatar.jpg
+Subject: =?utf-8?q?Message_from_=F0=9F=8F=9E=EF=B8=8F_Mefiscali?=
+MIME-Version: 1.0
+Date: Mon, 15 Jun 2020 16:02:22 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@tiscali.it; prefer-encrypt=mutual;
+	keydata=xjMEXtFRUBYJKwYBBAHaRw8BAQdA5sqHJqkWlveCgsNd0rtwtZrT1mmo1gwaGC5+WheYk5
+	nNHTxhbmRyZWFzLmxhdHRtYW5uQHRpc2NhbGkuaXQ+wosEEBYIADMCGQEFAl7RUXoCGwMECwkIBwYV
+	CAkKCwIDFgIBFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrAAaAEA4wssXeU2IXnowv
+	iu3zmcNzDgE4HdmW4RFyqJC6bgxXQA/3aTfE/PhQgZvi6RrKMvP4zygXpD9y+3ydIZP88Bp8kIzjgE
+	XtFRUBIKKwYBBAGXVQEFAQEHQKaAwlP0j9m0aYsCtO+qD9+foH0kiTN5BWDe5YcZrckVAwEIB8J4BB
+	gWCAAgBQJe0VF6AhsMFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrA1JgEAkscCQlps
+	h3ZxlLqBlbf2+85f4S4aGQfFPtIYEkKKhYEBAJbQulNNp9UarvhfyBiIdvkBVDcCnJZwzbORqp8RM0 gC
+Message-ID: <Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it>
+To: <shenauithz@testrun.org>
+From: =?utf-8?b?8J+Pnu+4jyBNZWZpc2NhbGk=?= <alice@tiscali.it>
+Content-Type: multipart/mixed; boundary="5uAmYQux1HZxxriijTjjKSp4DMoJwq"
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tiscali.it; s=smtp;
+	t=1592236943; bh=C3taz+zuSre1ko5Q5CzGPmbyrgegKYBClx/3Dv7t/Xw=;
+	h=Subject:Date:To:From;
+	b=LrcfLfrQoemOkHTQsqR8MExqNlx5KPYNFWhwlBWylvVc5GlmlhzqM6SAVKd0NVsKE
+	 gVRlBId5FvnlwoJ2WZnXaw/+3lWKilMTuzzQ1oFGvLnZ1XUaUEfuliIv+9NI79dJWX
+	 +S3jsSgzJMJc9+fO6s9bJsX1EHQ2a8GXNbwDtLXs=
+
+
+
+--------------I305M09060309060P_896715922369450--
+
+

test-data/message/yahoo_ndn.eml

@@ -0,0 +1,100 @@
+X-Atlas-Received: from 10.218.250.153 by atlas222.free.mail.ne1.yahoo.com with http; Wed, 10 Jun 2020 10:11:23 +0000
+Return-Path: <>
+Received: from 77.238.177.145 (EHLO sonic314-19.consmr.mail.ir2.yahoo.com)
+ by atlas222.free.mail.ne1.yahoo.com with SMTPs; Wed, 10 Jun 2020 10:11:23 +0000
+X-Originating-Ip: [77.238.177.145]
+Received-SPF: none (domain of sonic314-19.consmr.mail.ir2.yahoo.com does not designate permitted sender hosts)
+Authentication-Results: atlas222.free.mail.ne1.yahoo.com;
+ dkim=pass header.i=@yahoo.com header.s=@bounce;
+ spf=none smtp.mailfrom=sonic314-19.consmr.mail.ir2.yahoo.com;
+ dmarc=success(p=REJECT) header.from=yahoo.com;
+X-Apparently-To: alice@yahoo.com; Wed, 10 Jun 2020 10:11:23 +0000
+X-YMailISG: RT5ZnycWLDvIW52uqHS_EWNgl31NdJPyLLB2F4SYb1GCAoo9
+ pcninuVU5GDMBZykeMT4cSUt4ZqXxS5FdEeWJqtGIAtbEGbIL8Uhcoszqm4m
+ JuMJiQZwEE7W_fsS_9MUK5gZtMkhKkSnAuaeaOLKNYAwFZdBqA0uEYA5EmVf
+ EC9J4RGQ4hZvrMqMj_W.cj4pvbEC.pyirLxTfkICuUkZVguYoxG16y1EOJPw
+ B48fhXvF5ErU7WAHKxyRM3bMOg7b5pXHKn1dtRSVAXEuqBAQrWig1pePpYH1
+ wO54sYT7cgmdiFvfLY5rR7YcBzopmKJBycKzBVoRLCY4gvoNyTLPKx9o3AAz
+ WU4B7TGejDBElYSLpfnyvQg8wU27zzo2IVBZWUNztP0Ca8CQ07Y7TxUZAO.f
+ DNO5c7nd81PHMRDbSeaw1BTV2Yd9vlBc7syYmwGvtVBJQwRU7qPN.DpFO2jC
+ 9j9DytVhm5231gdBBRSzW78yG.VvaIdJgq_YViKNM9VxFseTz3Sjt3TaYznP
+ gAVq.MxpopNsSZf_tedwAhXDWyrjKsRPK.v2ANivmuWGPednniEaMYhxJ05M
+ _5SnJ.hAU.l6h3HCEfiU.SH390_3tZgYNfxCo4GPPFMfnNPmKa3.rgpChBCz
+ 9CRexJ8BSFyCEeAhuqQ8vSJfSuittJmXvS6Tk8Rxd9HUJAtKzZ.xCWZQ4tA6
+ Yp2aRG23_rK_C6hH8ArkkvbG.uVQTt6DltSX6avJLObBfIhBH0x64RoFjGee
+ vYXxM741Okm0jH7r79c8GhnAwas_bwfkaTW9e1nhYP0eyI36z_QwLYgOH3Mm
+ LrUcejpOMDR60QWDuDyRbWXOJdr3Q2K0ERhuAy6YnINq0sL3HX7t5wjsFLvp
+ _7Ri_eruTfIst4C7DZwERwui6aDSEAdF1Z8oZukBVmiyZsHmhJQCUik646iy
+ 3ASMR3lX7R3q2PBHQo2oC3qte8Fzz1FhKoMtfCGtIpeCazlkhDEJ6eTBSQ3R
+ Pe7M_GPiv3QNp7qu5CWHlzy6hWEKIkNwx.WRGYzfxkyJMmJm4UrhQYUfa4lG
+ Wb8n.mfYnS_KGYtzyRFNqAL0IGo.1MB9aG6qQk456Fz9GJgbHLWrMXVtyfrr
+ Uo7mKih8FCrdUKv5X6KBnpY0vvyoH5jrWyrvo3DW0bq_JvZ9U51JwUhoGY5U
+ c1t.yCSJbs8tnrGZHuUTOvouWzpCAJsk34AqRyH0wDJZQsAwBW5UZ3jx8ARA
+ FicoSqZCa4wEP9WaaXvfzFbmLW0-
+X-Originating-IP: [77.238.177.145]
+Received: from 10.217.135.165  (EHLO sonic314-19.consmr.mail.ir2.yahoo.com) (77.238.177.145)
+  by mta4277.mail.ne1.yahoo.com with SMTPS; Wed, 10 Jun 2020 10:11:21 +0000
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=bounce; t=1591783879; bh=4BmZzBC/nu0AJ9r0i0xNuCENks2KZcuXCbjSHdzbg9Y=; h=Date:From:To:Subject:From:Subject; b=lPxu8goOGOLVgnwbndfdptZ7zI5VEo0lSSr+ONGxwdtuhrySKDU6Sp41/g6jWbAiVPT1947j/B5wOlPfa5tv4XkWrGf0JCbT1I20ZJIkNfNwt4F0qPnbJAiHFIDPxcY68utjC9IgPWJd0cGqJNXbFwbJBu88rtrbMoInzLakh5I=
+Received: from sonic.gate.mail.ne1.yahoo.com by sonic314.consmr.mail.ir2.yahoo.com with HTTP; Wed, 10 Jun 2020 10:11:19 +0000
+Date: Wed, 10 Jun 2020 10:11:19 +0000
+From: MAILER-DAEMON@yahoo.com
+To: alice@yahoo.com
+Message-ID: <1713051795.39992.1591783879940@sonic314.consmr.mail.ir2.yahoo.com>
+Subject: Failure Notice
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+Content-Length: 3347
+
+Sorry, we were unable to deliver your message to the following address.
+
+<haeclirth.sinoenrat@yahoo.com>:
+554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com
+
+--- Below this line is a copy of the message.
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=s2048; t=1591783876; bh=kXD6TZuQDjqZf/AEAJ2HCX32Titkj3IywytG6GSm4yQ=; h=Date:From:To:Subject:References:From:Subject; b=PexnIBxnVSkyutv/jVn+Wlt5QPuVHnWleP3baWlvqkXaKR51pCaZIoGJggMEonitEeJkcYpgplBawEBz0hlGf63NqOHHkxUk6U1olwc0y9kj5kDH5lrORgXrf7U4z5t+i5n0II36MxbG9n/5tDRXoiabLFoWx//3O9x/ZJvVWPlq8RBFzVG8aoL2TkBQAVcqX/vW1f3WJuopaUYWB4AzR3TyuC2kVQPFqbPMk+G/VmyuZFmZesh1bSBva5hdKYLxES5v8hvTIDRqSYZrnZ4V67MqicZ8m229Xf4Za8qOE+a2Z+Vv5VrQ+CjjPZwRAcmKHkLY80VCSkpeL2R2YG4APg==
+X-YMail-OSG: 4spnw98VM1k_g51CM6oepdNEMiPFRtZ0ZG_zOBGIlhcNvS1mkzr8l8VTB0CY_Q0
+ dR52ikl7QVYESerRQgcGqBfLwKpem7i1XSrCl2HuyHeWzF6Gu5MqFPMCak.v8GyDXNO075NEwNt1
+ i18CJ29cEjiHthoamgmj0oqerAglglKRhTuuAFy4wUmZZm7VyvaW4wHUD1g7DeWGijQsCglSYMUK
+ CmoFcKsWOZBSYPMkp7iRwUp52pXHFin3qf4uQ27K_Sh.6s7KLAfWVkV7L_5AR3MyCPAVzm71.1yG
+ G7Vy5HSBgGMQ90B7VbcjOkCg3F4JNl4Z_P2ejV1KZ.tNoPLgO.FmsfFy1OXBGf3m2mDmRcuEO4K2
+ mTRhsjZf.2iiWpx02b3tY.oUtYrIBXBVIFPbTB9sBMn_9Z_qdVmO3gjD6gCPEBzuVvEO0eZIrgaw
+ EDTZt8Z9tSRDm1.4gV8LWQBYShF7XuMV0togiLYIO8s_iTHcTbhKhPlwxP.mxr06xcx_9kzReVTL
+ 9lB1FkB5Jm0WccWHGhLBqeMjGDoaNqPxLqJ.1tI58tLXsPoR6m1NFVEdzI1G.4AVBeXZ_9BjgUhm
+ KY33sEg.GwIjUlWWWuSyRZ1q1K2nqi1z29wH2R1Glmdmx0lyqfMg9Xe8HV7YZu2CuZ8SlDLLB.rX
+ NU4PwMsNfU6pK2HejQPsJuyOlI5Q824rXRF5xTLYKsYcQptoFXyLe6MXKW1ThBLQV2nWYDRs_V.e
+ SBmt22TfuOwu4Y5ju0sXmztZ8zpiIC8_rnAa5bVBEHxzkic64UZdukDX9V12Pk3G2sGYRyPTH472
+ wBX33JpDuq6BtrKr4FXjCLeVppARTHpiKM0jHMjmNf1bF0TvrcCsC9zAYtitAqgcGZNFETNuV3KM
+ 57XifdDEwUPOuww0ApSWO.iwP2POvIRBVlrxdgA8MbLmuuX4UxNCw23z1f7MVY6F3L60LUrX5GZO
+ aKaMmD1XTzx32J6c_TUmyuViT5vphqpEooTzHG2X7ALb4xC8yHlE4wDKyaEDARZ.8P2lO9T18oCz
+ OQvJjwDaLOkeAmo23yRMn70bYJK3tP9Z5cS1C0TE8PEtz4sd1syQUIZZ2g8JG_AQcE4lUZSZlIKN
+ AHjB8h8Uin35zKe0Le1DBjdQUmpgAETlmYE7V0nJDEmagB3dtpbokgRBuuBfhXlFpxHcnAmBFFFm
+ XOSLWEPnmxu2o8CCjjz3QUBy2fr3EI_D2VFpy..MuZgRwtES.l24m_95xtQxI28R4SWZN6LsS_rr
+ 1S33BJCCCAfXtCAFzCfz5.qSzHRYbLdY5do6yKj0pPLQTUTjlMwmCUGPcSJhsyxkkEVIK1W_Z16R
+ ZRls-
+Received: from sonic.gate.mail.ne1.yahoo.com by sonic314.consmr.mail.ir2.yahoo.com with HTTP; Wed, 10 Jun 2020 10:11:16 +0000
+Date: Wed, 10 Jun 2020 10:11:12 +0000 (UTC)
+From: Delta Chat Test <alice@yahoo.com>
+To: "haeclirth.sinoenrat@yahoo.com" <haeclirth.sinoenrat@yahoo.com>
+Message-ID: <1680295672.3657931.1591783872936@mail.yahoo.com>
+Subject: test
+MIME-Version: 1.0
+Content-Type: multipart/alternative; 
+	boundary="----=_Part_3657930_145367320.1591783872935"
+References: <1680295672.3657931.1591783872936.ref@mail.yahoo.com>
+X-Mailer: WebService/1.1.16072 YMailNorrin Mozilla/5.0 (X11; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0
+Content-Length: 494
+
+------=_Part_3657930_145367320.1591783872935
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+test
+
+------=_Part_3657930_145367320.1591783872935
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+<html><head></head><body><div class="yahoo-style-wrap" style="font-family:Helvetica Neue, Helvetica, Arial, sans-serif;font-size:16px;"><div dir="ltr" data-setdir="false">test<br></div></div></body></html>
+------=_Part_3657930_145367320.1591783872935--