bump version to 1.41

fix test_decode_account_bad_scheme

.circleci/config.yml

@@ -138,12 +138,6 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests_rust:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_rust.sh
-    
   remote_tests_python:
     machine: true
     steps:
@@ -178,11 +172,6 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
-
       - remote_tests_python:
           filters:
             tags:
@@ -191,8 +180,9 @@ workflows:
       - remote_python_packaging:
           requires: 
              - remote_tests_python
-             - remote_tests_rust 
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 
@@ -201,6 +191,8 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -212,6 +204,8 @@ workflows:
 
       - build_doxygen:
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -0,0 +1,101 @@
+name: Rust CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.45.0
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.45.0
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all
+
+
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [1.45.0]
+        experimental: [false]
+        include:
+          - os: ubuntu-latest
+            rust: nightly
+            experimental: true
+          - os: windows-latest
+            rust: nightly
+            experimental: true
+          - os: macOS-latest
+            rust: nightly
+            experimental: true
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-19
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

ASYNC-API-TODO.txt

@@ -1,54 +0,0 @@
-
-Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
-
-- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
-
-APIs:
-    dc_context_new                 # opens the database
-    dc_open                             # FFI only
-      -> drop it and move parameters to dc_context_new()
-
-    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
-    dc_imex NEVER goes through the job system
-    dc_imex import_backup needs to ensure dc_stop_jobs()
-
-    dc_start_io                # start smtp/imap and job handling subsystems
-    dc_stop_io                 # stop smtp/imap and job handling subsystems
-    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
-
-    dc_close                            # FFI only
-      -> can be dropped
-    dc_context_unref
-
-    for ios share-extension:
-    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
-        0: message was sent
-        1: message failed to go out, is queued as a job to be retried later
-        2: message permanently failed?
-
-EVENT handling:
-    start a callback thread and call get_next_event() which is BLOCKING
-    it's fine to start this callback thread later, it will see all events.
-    Note that the core infinitely fills the internal queue if you never drain it.
-
-    FFI-get_next_event() returns NULL if the context is unrefed already?
-
-   sidenote: how python's callback thread does it currently:
-         CB-thread runs this while loop:
-         while not QUITFLAG:
-             ev = context.get_next_event( )
-             ...
-         So in order to shutdown properly one has to set QUITFLAG
-         before calling dc_stop_jobs() and dc_context_unref
-
-    event API:
-        get_data1_int
-        get_data2_int
-        get_data3_str
-
-
-- userdata likely only used for the callbacks, likely can be dropped, needs verification
-
-
-- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
-

CHANGELOG.md

@@ -1,5 +1,115 @@
 # Changelog 
 
+## 1.41.0
+
+- new apis to initiate video chats #1718 #1735
+
+- new apis `dc_msg_get_ephemeral_timer()`
+  and `dc_msg_get_ephemeral_timestamp()`
+
+- improve IMAP handling #1703 #1704
+
+- improve ephemeral messages #1696 #1705
+
+- mark location-messages as auto-generated #1715
+
+- multi-device avatar-sync #1716 #1717
+
+- improve python bindings #1732 #1733 #1738 #1769
+
+- Allow http scheme for DCACCOUNT urls #1770
+
+- more fixes #1702 #1706 #1707 #1710 #1719 #1721
+  #1723 #1734 #1740 #1744 #1748 #1760 #1766
+
+- refactorings #1712 #1714 #1757
+
+- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
+
+
+## 1.40.0
+
+- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
+
+- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
+
+- improve idle #1690 #1688
+
+- fix message processing issues by sequential processing #1694
+
+- refactorings #1670 #1673
+
+
+## 1.39.0
+
+- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
+
+- fix potential panics, update dependencies #1650 #1655
+
+
+## 1.38.0
+
+- fix sorting, esp. for multi-device
+
+
+## 1.37.0
+
+- improve ndn heuristics #1630
+
+- get oauth2 authorizer from provider-db #1641
+
+- removed linebreaks and spaces from generated qr-code #1631
+
+- more fixes #1633 #1635 #1636 #1637
+
+
+## 1.36.0
+
+- parse ndn (network delivery notification) reports
+  and report failed messages as such #1552 #1622 #1630
+
+- add oauth2 support for gsuite domains #1626
+
+- read image orientation from exif before recoding #1619
+
+- improve logging #1593 #1598
+
+- improve python and bot bindings #1583 #1609
+
+- improve imap logout #1595
+
+- fix sorting #1600 #1604
+
+- fix qr code generation #1631
+
+- update rustcrypto releases #1603
+
+- refactorings #1617
+
+
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
 ## 1.34.0
 
 - new api for io, thread and event handling #1356,

Cargo.toml

@@ -1,23 +1,23 @@
 [package]
 name = "deltachat"
-version = "1.34.0"
+version = "1.41.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
 [profile.release]
-# lto = true
+lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
 async-smtp = { version = "0.3" }
@@ -25,42 +25,42 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.3.1"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.0", features = ["unstable"] }
-base64 = "0.11"
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
+kamadak-exif = "0.5"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.22", features = ["bundled"] }
-r2d2_sqlite = "0.15.0"
+rusqlite = { version = "0.23", features = ["bundled"] }
+r2d2_sqlite = "0.16.0"
 r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
+strum = "0.18.0"
+strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
-itertools = "0.8.0"
-image-meta = "0.1.0"
-quick-xml = "0.17.1"
+itertools = "0.9.0"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
 mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
+async-std-resolver = "0.19.5"
 
-pretty_env_logger = { version = "0.3.1", optional = true }
+pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
@@ -69,8 +69,8 @@ ansi_term = { version = "0.12.1", optional = true }
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
 async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
 smol = "0.1.10"
 
@@ -98,5 +98,3 @@ repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 
-[patch.crates-io]
-smol = { git = "https://github.com/dignifiedquire/smol-1", branch = "isolate-nix" }

README.md

@@ -123,6 +123,7 @@ Language bindings are available for:
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
 - [Go](https://github.com/hugot/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

appveyor.yml

@@ -1,19 +0,0 @@
-environment:
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-19
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - rustc -vV
-  - cargo -vV
-
-build: false
-
-test_script:
-  - cargo test --release --all
-
-cache:
-  - target
-  - C:\Users\appveyor\.cargo\registry

ci_scripts/docker-coredeps/Dockerfile

@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust nightly
+# Install Rust
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.34.0"
+version = "1.41.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/build.rs

@@ -19,10 +19,10 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
+        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();

deltachat-ffi/deltachat.h

@@ -262,17 +262,25 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
  *                    `blobdir`; the original image will not be needed anymore.
  *                    NULL to remove the avatar.
- *                    It is planned for future versions
- *                    to send this image together with the next messages.
+ *                    As for `displayname` and `selfstatus`, also the avatar is sent to the recipients.
+ *                    To save traffic, however, the avatar is attached only as needed
+ *                    and also recoded to a reasonable size.
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
  *                    1=send and request read receipts (default)
+ * - `bcc_self`     = 0=do not send a copy of outgoing messages to self (default),
+ *                    1=send a copy of outgoing messages to self.
+ *                    Sending messages to self is needed for a proper multi-account setup,
+ *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
  * - `inbox_watch`  = 1=watch `INBOX`-folder for changes (default),
- *                    0=do not watch the `INBOX`-folder
+ *                    0=do not watch the `INBOX`-folder,
+ *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes (default),
- *                    0=do not watch the `Sent`-folder
+ *                    0=do not watch the `Sent`-folder,
+ *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `mvbox_watch`  = 1=watch `DeltaChat`-folder for changes (default),
- *                    0=do not watch the `DeltaChat`-folder
+ *                    0=do not watch the `DeltaChat`-folder,
+ *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `mvbox_move`   = 1=heuristically detect chat-messages
  *                    and move them to the `DeltaChat`-folder,
  *                    0=do not move chat-messages
@@ -307,9 +315,14 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    DC_MEDIA_QUALITY_WORSE (1)
  *                    allow worse images/videos/voice quality to gain smaller sizes,
  *                    suitable for providers or areas known to have a bad connection.
- *                    In contrast to other options, the implementation of this option is currently up to the UIs;
- *                    this may change in future, however,
- *                    having the option in the core allows provider-specific-defaults already today.
+ *                    The library uses the `media_quality` setting to use different defaults
+ *                    for recoding images sent with type DC_MSG_IMAGE.
+ *                    If needed, recoding other file types is up to the UI.
+ * - `webrtc_instance` = webrtc instance to use for videochats in the form
+ *                    `[basicwebrtc:]https://example.com/subdir#roomname=$ROOM`
+ *                    if the url is prefixed by `basicwebrtc`, the server is assumed to be of the type
+ *                    https://github.com/cracker0dks/basicwebrtc which some UIs have native support for.
+ *                    If no type is prefixed, the videochat is handled completely in a browser.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -528,9 +541,20 @@ int             dc_is_io_running(const dc_context_t* context);
 void            dc_stop_io(dc_context_t* context);
 
 /**
- * This function can be called whenever there is a hint
- * that the network is available again.
- * The library will try to send pending messages out.
+ * This function should be called when there is a hint
+ * that the network is available again,
+ * eg. as a response to system event reporting network availability.
+ * The library will try to send pending messages out immediately.
+ *
+ * Moreover, to have a reliable state
+ * when the app comes to foreground with network available,
+ * it may be reasonable to call the function also at that moment.
+ *
+ * It is okay to call the function unconditionally when there is
+ * network available, however, calling the function
+ * _without_ having network may interfere with the backoff algorithm
+ * and will led to let the jobs fail faster, with fewer retries
+ * and may avoid messages being sent out.
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
@@ -754,6 +778,15 @@ uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t ch
  * dc_msg_unref(msg);
  * ~~~
  *
+ * If you send images with the DC_MSG_IMAGE type,
+ * they will be recoded to a reasonable size before sending, if possible
+ * (cmp the dc_set_config()-option `media_quality`).
+ * If that fails, is not possible, or the image is already small enough, the image is sent as original.
+ * If you want images to be always sent as the original file, use the DC_MSG_FILE type.
+ *
+ * Videos and other file types are currently not recoded by the library,
+ * with dc_prepare_msg(), however, you can do that from the UI.
+ *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id Chat ID to send the message to.
@@ -806,6 +839,42 @@ uint32_t        dc_send_msg_sync                  (dc_context_t* context, uint32
 uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t chat_id, const char* text_to_send);
 
 
+/**
+ * Send invitation to a videochat.
+ *
+ * This function reads the `webrtc_instance` config value,
+ * may check that the server is working in some way
+ * and creates a unique room for this chat, if needed doing a TOKEN roundtrip for that.
+ *
+ * After that, the function sends out a message that contains information to join the room:
+ *
+ * - To allow non-delta-clients to join the chat,
+ *   the message contains a text-area with some descriptive text
+ *   and a url that can be opened in a supported browser to join the videochat
+ *
+ * - delta-clients can get all information needed from
+ *   the message object, using eg.
+ *   dc_msg_get_videochat_url() and check dc_msg_get_viewtype() for #DC_MSG_VIDEOCHAT_INVITATION
+ *
+ * dc_send_videochat_invitation() is blocking and may take a while,
+ * so the UIs will typically call the function from within a thread.
+ * Moreover, UIs will typically enter the room directly without an additional click on the message,
+ * for this purpose, the function returns the message-id directly.
+ *
+ * As for other messages sent, this function
+ * sends the event #DC_EVENT_MSGS_CHANGED on succcess, the message has a delivery state, and so on.
+ * The recipient will get noticed by the call as usual by #DC_EVENT_INCOMING_MSG or #DC_EVENT_MSGS_CHANGED,
+ * However, UIs might some things differently, eg. play a different sound.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param chat_id The chat to start a videochat for.
+ * @return The id if the message sent out
+ *     or 0 for errors.
+ */
+uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
+
+
 /**
  * Save a draft for a chat in the database.
  *
@@ -949,6 +1018,7 @@ dc_msg_t*       dc_get_draft                 (dc_context_t* context, uint32_t ch
  * @param chat_id The chat ID of which the messages IDs should be queried.
  * @param flags If set to DC_GCM_ADDDAYMARKER, the marker DC_MSG_ID_DAYMARKER will
  *     be added before each day (regarding the local timezone).  Set this to 0 if you do not want this behaviour.
+ *     To get the concrete time of the marker, use dc_array_get_timestamp().
  * @param marker1before An optional message ID.  If set, the id DC_MSG_ID_MARKER1 will be added just
  *   before the given ID in the returned array.  Set this to 0 if you do not want this behaviour.
  * @return Array of message IDs, must be dc_array_unref()'d when no longer used.
@@ -1143,6 +1213,16 @@ void            dc_delete_chat               (dc_context_t* context, uint32_t ch
  */
 dc_array_t*     dc_get_chat_contacts         (dc_context_t* context, uint32_t chat_id);
 
+/**
+ * Get the chat's ephemeral message timer.
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param chat_id The chat ID.
+ *
+ * @return ephemeral timer value in seconds, 0 if the timer is disabled or if there is an error
+ */
+uint32_t dc_get_chat_ephemeral_timer (dc_context_t* context, uint32_t chat_id);
 
 /**
  * Search messages containing the given query string.
@@ -1275,6 +1355,21 @@ int             dc_remove_contact_from_chat  (dc_context_t* context, uint32_t ch
  */
 int             dc_set_chat_name             (dc_context_t* context, uint32_t chat_id, const char* name);
 
+/**
+ * Set the chat's ephemeral message timer.
+ *
+ * This timer is applied to all messages in a chat and starts when the
+ * message is read. The setting is synchronized to all clients
+ * participating in a chat.
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param chat_id The chat ID to set the ephemeral message timer for.
+ * @param timer The timer value in seconds or 0 to disable the timer.
+ *
+ * @return 1=success, 0=error
+ */
+int dc_set_chat_ephemeral_timer (dc_context_t* context, uint32_t chat_id, uint32_t timer);
 
 /**
  * Set group profile image.
@@ -2257,17 +2352,6 @@ int              dc_array_is_independent     (const dc_array_t* array, size_t in
 int              dc_array_search_id          (const dc_array_t* array, uint32_t needle, size_t* ret_index);
 
 
-/**
- * Get raw pointer to the data.
- *
- * @memberof dc_array_t
- * @param array The array object.
- * @return Raw pointer to the array. You MUST NOT free the data. You MUST NOT access the data beyond the current item count.
- *     It is not possible to enlarge the array this way.  Calling any other dc_array*()-function may discard the returned pointer.
- */
-const uint32_t*  dc_array_get_raw            (const dc_array_t* array);
-
-
 /**
  * @class dc_chatlist_t
  *
@@ -2782,6 +2866,9 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_DELIVERED.
  * - DC_STATE_OUT_MDN_RCVD (28) - Outgoing message read by the recipient (two checkmarks; this requires goodwill on the receiver's side)
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_READ.
+ *   Also messages already read by some recipients
+ *   may get into the state DC_STATE_OUT_FAILED at a later point,
+ *   eg. when in a group, delivery fails for some recipients.
  *
  * If you just want to check if a message is sent or not, please use dc_msg_is_sent() which regards all states accordingly.
  *
@@ -2980,6 +3067,32 @@ int             dc_msg_get_duration           (const dc_msg_t* msg);
 int             dc_msg_get_showpadlock        (const dc_msg_t* msg);
 
 
+/**
+ * Get ephemeral timer duration for message.
+ *
+ * To check if the timer is started and calculate remaining time,
+ * use dc_msg_get_ephemeral_timestamp().
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return Duration in seconds, or 0 if no timer is set.
+ */
+uint32_t        dc_msg_get_ephemeral_timer    (const dc_msg_t* msg);
+
+/**
+ * Get timestamp of ephemeral message removal.
+ *
+ * If returned value is non-zero, you can calculate the * fraction of
+ * time remaining by divinding the difference between the current timestamp
+ * and this timestamp by dc_msg_get_ephemeral_timer().
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return Time of message removal, 0 if the timer is not started.
+ */
+int64_t          dc_msg_get_ephemeral_timestamp (const dc_msg_t* msg);
+
+
 /**
  * Get a summary for a message.
  *
@@ -3163,6 +3276,57 @@ int             dc_msg_is_setupmessage        (const dc_msg_t* msg);
 char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
 
 
+/**
+ * Get url of a videochat invitation.
+ *
+ * Videochat invitations are sent out using dc_send_videochat_invitation()
+ * and dc_msg_get_viewtype() returns #DC_MSG_VIDEOCHAT_INVITATION for such invitations.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return If the message contains a videochat invitation,
+ *     the url of the invitation is returned.
+ *     If the message is no videochat invitation, NULL is returned.
+ *     Must be released using dc_str_unref() when done.
+ */
+char* dc_msg_get_videochat_url (const dc_msg_t* msg);
+
+
+/**
+ * Get type of videochat.
+ *
+ * Calling this functions only makes sense for messages of type #DC_MSG_VIDEOCHAT_INVITATION,
+ * in this case, if "basic webrtc" as of https://github.com/cracker0dks/basicwebrtc was used to initiate the videochat,
+ * dc_msg_get_videochat_type() returns DC_VIDEOCHATTYPE_BASICWEBRTC.
+ * "basic webrtc" videochat may be processed natively by the app
+ * whereas for other urls just the browser is opened.
+ *
+ * The videochat-url can be retrieved using dc_msg_get_videochat_url().
+ * To check if a message is a videochat invitation at all, check the message type for #DC_MSG_VIDEOCHAT_INVITATION.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return Type of the videochat as of DC_VIDEOCHATTYPE_BASICWEBRTC or DC_VIDEOCHATTYPE_UNKNOWN.
+ *
+ * Example:
+ * ~~~
+ * if (dc_msg_get_viewtype(msg) == DC_MSG_VIDEOCHAT_INVITATION) {
+ *   if (dc_msg_get_videochat_type(msg) == DC_VIDEOCHATTYPE_BASICWEBRTC) {
+ *       // videochat invitation that we ship a client for
+ *   } else {
+ *       // use browser for videochat, just open the url
+ *   }
+ * } else {
+ *    // not a videochat invitation
+ * }
+ * ~~~
+ */
+int dc_msg_get_videochat_type (const dc_msg_t* msg);
+
+#define DC_VIDEOCHATTYPE_UNKNOWN     0
+#define DC_VIDEOCHATTYPE_BASICWEBRTC 1
+
+
 /**
  * Set the text of a message object.
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
@@ -3698,6 +3862,18 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define DC_MSG_FILE      60
 
+
+/**
+ * Message indicating an incoming or outgoing videochat.
+ * The message was created via dc_send_videochat_invitation() on this or a remote device.
+ *
+ * Typically, such messages are rendered differently by the UIs,
+ * eg. contain a button to join the videochat.
+ * The url for joining can be retrieved using dc_msg_get_videochat_url().
+ */
+#define DC_MSG_VIDEOCHAT_INVITATION 70
+
+
 /**
  * @}
  */
@@ -4055,14 +4231,12 @@ void dc_event_unref(dc_event_t* event);
  * Network errors should be reported to users in a non-disturbing way,
  * however, as network errors may come in a sequence,
  * it is not useful to raise each an every error to the user.
- * For this purpose, data1 is set to 1 if the error is probably worth reporting.
  *
  * Moreover, if the UI detects that the device is offline,
  * it is probably more useful to report this to the user
  * instead of the string from data2.
  *
- * @param data1 (int) 1=first/new network error, should be reported the user;
- *     0=subsequent network error, should be logged only
+ * @param data1 0
  * @param data2 (char*) Error string, always set, never NULL.
  */
 #define DC_EVENT_ERROR_NETWORK            401
@@ -4117,8 +4291,9 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
- * DC_STATE_OUT_FAILED, see dc_msg_get_state().
+ * A single message could not be sent.
+ * State changed from DC_STATE_OUT_PENDING, DC_STATE_OUT_DELIVERED or DC_STATE_OUT_MDN_RCVD
+ * to DC_STATE_OUT_FAILED, see dc_msg_get_state().
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -4147,6 +4322,11 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_CHAT_MODIFIED            2020
 
+/**
+ * Chat ephemeral timer changed.
+ */
+#define DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED 2021
+
 
 /**
  * Contact(s) created, renamed, verified, blocked or deleted.
@@ -4283,7 +4463,7 @@ void dc_event_unref(dc_event_t* event);
  */
 
 /**
- * Prover works out-of-the-box.
+ * Provider works out-of-the-box.
  * This provider status is returned for provider where the login
  * works by just entering the name or the email-address.
  *
@@ -4427,12 +4607,31 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_DEVICE_MESSAGES_HINT       70
 #define DC_STR_WELCOME_MESSAGE            71
 #define DC_STR_UNKNOWN_SENDER_FOR_CHAT    72
-#define DC_STR_COUNT                      72
+#define DC_STR_SUBJECT_FOR_NEW_CONTACT    73
+#define DC_STR_FAILED_SENDING_TO          74
+#define DC_STR_EPHEMERAL_DISABLED         75
+#define DC_STR_EPHEMERAL_SECONDS          76
+#define DC_STR_EPHEMERAL_MINUTE           77
+#define DC_STR_EPHEMERAL_HOUR             78
+#define DC_STR_EPHEMERAL_DAY              79
+#define DC_STR_EPHEMERAL_WEEK             80
+#define DC_STR_EPHEMERAL_FOUR_WEEKS       81
+#define DC_STR_VIDEOCHAT_INVITATION       82
+#define DC_STR_VIDEOCHAT_INVITE_MSG_BODY  83
+
+#define DC_STR_COUNT                      83
 
 /*
  * @}
  */
 
+#ifdef PY_CFFI_INC
+/* Helper utility to locate the header file when building python bindings. */
+char* _dc_header_file_location(void) {
+    return __FILE__;
+}
+#endif
+
 
 #ifdef __cplusplus
 }

deltachat-ffi/src/dc_array.rs

@@ -1,46 +1,56 @@
+use crate::chat::ChatItem;
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
+use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
+    MsgIds(Vec<MsgId>),
+    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub fn new(capacity: usize) -> Self {
-        dc_array_t::Uint(Vec::with_capacity(capacity))
-    }
-
-    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
-    pub fn new_locations(capacity: usize) -> Self {
-        dc_array_t::Locations(Vec::with_capacity(capacity))
-    }
-
-    pub fn add_id(&mut self, item: u32) {
-        if let Self::Uint(array) = self {
-            array.push(item);
-        } else {
-            panic!("Attempt to add id to array of other type");
-        }
-    }
-
-    pub fn add_location(&mut self, location: Location) {
-        if let Self::Locations(array) = self {
-            array.push(location)
-        } else {
-            panic!("Attempt to add a location to array of other type");
-        }
-    }
-
-    pub fn get_id(&self, index: usize) -> u32 {
+    pub(crate) fn get_id(&self, index: usize) -> u32 {
         match self {
+            Self::MsgIds(array) => array[index].to_u32(),
+            Self::Chat(array) => match array[index] {
+                ChatItem::Message { msg_id } => msg_id.to_u32(),
+                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
+                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
+            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index] as u32,
+            Self::Uint(array) => array[index],
         }
     }
 
-    pub fn get_location(&self, index: usize) -> &Location {
+    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(array) => array.get(index).and_then(|item| match item {
+                ChatItem::Message { .. } => None,
+                ChatItem::Marker1 { .. } => None,
+                ChatItem::DayMarker { timestamp } => Some(*timestamp),
+            }),
+            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(_) => None,
+            Self::Locations(array) => array
+                .get(index)
+                .and_then(|location| location.marker.as_deref()),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -48,55 +58,18 @@ impl dc_array_t {
         }
     }
 
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Locations(array) => array.is_empty(),
-            Self::Uint(array) => array.is_empty(),
-        }
-    }
-
     /// Returns the number of elements in the array.
-    pub fn len(&self) -> usize {
+    pub(crate) fn len(&self) -> usize {
         match self {
+            Self::MsgIds(array) => array.len(),
+            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub fn clear(&mut self) {
-        match self {
-            Self::Locations(array) => array.clear(),
-            Self::Uint(array) => array.clear(),
-        }
-    }
-
-    pub fn search_id(&self, needle: u32) -> Option<usize> {
-        if let Self::Uint(array) = self {
-            for (i, &u) in array.iter().enumerate() {
-                if u == needle {
-                    return Some(i);
-                }
-            }
-            None
-        } else {
-            panic!("Attempt to search for id in array of other type");
-        }
-    }
-
-    pub fn sort_ids(&mut self) {
-        if let dc_array_t::Uint(v) = self {
-            v.sort();
-        } else {
-            panic!("Attempt to sort array of something other than uints");
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const u32 {
-        if let dc_array_t::Uint(v) = self {
-            v.as_ptr()
-        } else {
-            panic!("Attempt to convert array of something other than uints to raw");
-        }
+    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
+        (0..self.len()).find(|i| self.get_id(*i) == needle)
     }
 }
 
@@ -106,6 +79,18 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
+impl From<Vec<MsgId>> for dc_array_t {
+    fn from(array: Vec<MsgId>) -> Self {
+        dc_array_t::MsgIds(array)
+    }
+}
+
+impl From<Vec<ChatItem>> for dc_array_t {
+    fn from(array: Vec<ChatItem>) -> Self {
+        dc_array_t::Chat(array)
+    }
+}
+
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -118,12 +103,11 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let mut arr = dc_array_t::new(7);
-        assert!(arr.is_empty());
+        let arr: dc_array_t = Vec::<u32>::new().into();
+        assert!(arr.len() == 0);
 
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
 
         assert_eq!(arr.len(), 1000);
 
@@ -131,31 +115,15 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        arr.clear();
-
-        assert!(arr.is_empty());
-
-        arr.add_id(13);
-        arr.add_id(7);
-        arr.add_id(666);
-        arr.add_id(0);
-        arr.add_id(5000);
-
-        arr.sort_ids();
-
-        assert_eq!(arr.get_id(0), 0);
-        assert_eq!(arr.get_id(1), 7);
-        assert_eq!(arr.get_id(2), 13);
-        assert_eq!(arr.get_id(3), 666);
+        assert_eq!(arr.search_id(10), Some(8));
+        assert_eq!(arr.search_id(1), None);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let mut arr = dc_array_t::new(7);
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/lib.rs

@@ -1,3 +1,4 @@
+#![deny(clippy::all)]
 #![allow(
     non_camel_case_types,
     non_snake_case,
@@ -27,6 +28,7 @@ use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
+use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock::StockMessage;
@@ -126,7 +128,7 @@ pub unsafe extern "C" fn dc_set_config(
         // When ctx.set_config() fails it already logged the error.
         // TODO: Context::set_config() should not log this
         Ok(key) => block_on(async move {
-            ctx.set_config(key, to_opt_string_lossy(value).as_ref().map(|x| x.as_str()))
+            ctx.set_config(key, to_opt_string_lossy(value).as_deref())
                 .await
                 .is_ok() as libc::c_int
         }),
@@ -285,7 +287,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     }
     let ctx = &*context;
 
-    block_on({ ctx.start_io() })
+    block_on(ctx.start_io())
 }
 
 #[no_mangle]
@@ -295,7 +297,7 @@ pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c
     }
     let ctx = &*context;
 
-    block_on({ ctx.is_io_running() }) as libc::c_int
+    block_on(ctx.is_io_running()) as libc::c_int
 }
 
 #[no_mangle]
@@ -349,7 +351,8 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | Event::MsgDelivered { chat_id, .. }
         | Event::MsgFailed { chat_id, .. }
         | Event::MsgRead { chat_id, .. }
-        | Event::ChatModified(chat_id) => chat_id.to_u32() as libc::c_int,
+        | Event::ChatModified(chat_id)
+        | Event::ChatEphemeralTimerModified { chat_id, .. } => chat_id.to_u32() as libc::c_int,
         Event::ContactsChanged(id) | Event::LocationChanged(id) => {
             let id = id.unwrap_or_default();
             id as libc::c_int
@@ -399,6 +402,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | Event::MsgRead { msg_id, .. } => msg_id.to_u32() as libc::c_int,
         Event::SecurejoinInviterProgress { progress, .. }
         | Event::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
+        Event::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
     }
 }
 
@@ -439,7 +443,8 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | Event::ConfigureProgress(_)
         | Event::ImexProgress(_)
         | Event::SecurejoinInviterProgress { .. }
-        | Event::SecurejoinJoinerProgress { .. } => ptr::null_mut(),
+        | Event::SecurejoinJoinerProgress { .. }
+        | Event::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         Event::ImexFileWritten(file) => {
             let data2 = file.to_c_string().unwrap_or_default();
             data2.into_raw()
@@ -482,7 +487,7 @@ pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *
     events
         .recv_sync()
         .map(|ev| Box::into_raw(Box::new(ev)))
-        .unwrap_or_else(|| ptr::null_mut())
+        .unwrap_or_else(ptr::null_mut)
 }
 
 #[no_mangle]
@@ -554,14 +559,7 @@ pub unsafe extern "C" fn dc_get_chatlist(
     let qi = if query_id == 0 { None } else { Some(query_id) };
 
     block_on(async move {
-        match chatlist::Chatlist::try_load(
-            &ctx,
-            flags as usize,
-            qs.as_ref().map(|x| x.as_str()),
-            qi,
-        )
-        .await
-        {
+        match chatlist::Chatlist::try_load(&ctx, flags as usize, qs.as_deref(), qi).await {
             Ok(list) => {
                 let ffi_list = ChatlistWrapper { context, list };
                 Box::into_raw(Box::new(ffi_list))
@@ -712,6 +710,25 @@ pub unsafe extern "C" fn dc_send_text_msg(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_send_videochat_invitation(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_send_videochat_invitation()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        chat::send_videochat_invitation(&ctx, ChatId::new(chat_id))
+            .await
+            .map(|msg_id| msg_id.to_u32())
+            .unwrap_or_log_default(&ctx, "Failed to send video chat invitation")
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -752,13 +769,9 @@ pub unsafe extern "C" fn dc_add_device_msg(
     };
 
     block_on(async move {
-        chat::add_device_msg(
-            &ctx,
-            to_opt_string_lossy(label).as_ref().map(|x| x.as_str()),
-            msg,
-        )
-        .await
-        .unwrap_or_log_default(&ctx, "Failed to add device message")
+        chat::add_device_msg(&ctx, to_opt_string_lossy(label).as_deref(), msg)
+            .await
+            .unwrap_or_log_default(&ctx, "Failed to add device message")
     })
     .to_u32()
 }
@@ -841,14 +854,11 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
     };
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_msgs(&ctx, ChatId::new(chat_id), flags, marker_flag)
                 .await
-                .iter()
-                .map(|msg_id| msg_id.to_u32())
-                .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+                .into(),
+        ))
     })
 }
 
@@ -977,7 +987,7 @@ pub unsafe extern "C" fn dc_get_chat_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_media(
                 &ctx,
                 ChatId::new(chat_id),
@@ -986,11 +996,8 @@ pub unsafe extern "C" fn dc_get_chat_media(
                 or_msg_type3,
             )
             .await
-            .iter()
-            .map(|msg_id| msg_id.to_u32())
-            .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+            .into(),
+        ))
     })
 }
 
@@ -1296,6 +1303,49 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    // Timer value 0 is returned in the rare case of a database error,
+    // but it is not dangerous since it is only meant to be used as a
+    // default when changing the value. Such errors should not be
+    // ignored when ephemeral timer value is used to construct
+    // message headers.
+    block_on(async move { ChatId::new(chat_id).get_ephemeral_timer(ctx).await })
+        .log_err(ctx, "Failed to get ephemeral timer")
+        .unwrap_or_default()
+        .to_u32()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    timer: u32,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_set_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .set_ephemeral_timer(ctx, EphemeralTimer::from_u32(timer))
+            .await
+            .log_err(ctx, "Failed to set ephemeral timer")
+            .is_ok() as libc::c_int
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_msg_info(
     context: *mut dc_context_t,
@@ -1829,7 +1879,11 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
     }
     let ctx = &*context;
 
-    block_on({ location::send_locations_to_chat(&ctx, ChatId::new(chat_id), seconds as i64) });
+    block_on(location::send_locations_to_chat(
+        &ctx,
+        ChatId::new(chat_id),
+        seconds as i64,
+    ));
 }
 
 #[no_mangle]
@@ -1987,7 +2041,7 @@ pub unsafe extern "C" fn dc_array_get_timestamp(
         return 0;
     }
 
-    (*array).get_location(index).timestamp
+    (*array).get_timestamp(index).unwrap_or_default()
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_chat_id(
@@ -2034,7 +2088,7 @@ pub unsafe extern "C" fn dc_array_get_marker(
         return std::ptr::null_mut(); // NULL explicitly defined as "no markers"
     }
 
-    if let Some(s) = &(*array).get_location(index).marker {
+    if let Some(s) = (*array).get_marker(index) {
         s.strdup()
     } else {
         std::ptr::null_mut()
@@ -2062,16 +2116,6 @@ pub unsafe extern "C" fn dc_array_search_id(
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_array_get_raw(array: *const dc_array_t) -> *const u32 {
-    if array.is_null() {
-        eprintln!("ignoring careless call to dc_array_get_raw()");
-        return ptr::null_mut();
-    }
-
-    (*array).as_ptr()
-}
-
 // Return the independent-state of the location at the given index.
 // Independent locations do not belong to the track of the user.
 // Returns 1 if location belongs to the track of the user,
@@ -2651,6 +2695,26 @@ pub unsafe extern "C" fn dc_msg_get_showpadlock(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.get_showpadlock() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_ephemeral_timer(msg: *mut dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_ephemeral_timer()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_ephemeral_timer()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_ephemeral_timestamp(msg: *mut dc_msg_t) -> i64 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_ephemeral_timer()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_ephemeral_timestamp()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_summary(
     msg: *mut dc_msg_t,
@@ -2775,6 +2839,31 @@ pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.is_setupmessage().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_videochat_url()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+
+    ffi_msg
+        .message
+        .get_videochat_url()
+        .unwrap_or_default()
+        .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_videochat_type(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_videochat_type()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_videochat_type().unwrap_or_default() as i32
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_setupcodebegin(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2812,7 +2901,7 @@ pub unsafe extern "C" fn dc_msg_set_file(
     let ffi_msg = &mut *msg;
     ffi_msg.message.set_file(
         to_string_lossy(file),
-        to_opt_string_lossy(filemime).as_ref().map(|x| x.as_str()),
+        to_opt_string_lossy(filemime).as_deref(),
     )
 }
 

deltachat-ffi/src/string.rs

@@ -105,8 +105,9 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        CString::new(self.as_ref().as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         })
     }
 
@@ -122,8 +123,9 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         }),
         None => Err(CStringError::NotUnicode),
     }

draft/msgwork_new_imap_jobs.rst

@@ -0,0 +1,66 @@
+
+simplify/streamline mark-seen/delete/move/send-mdn job handling
+---------------------------------------------------------------
+
+Idea: Introduce a new "msgwork" sql table that looks very
+much like the jobs table but has a primary key "msgid"
+and no job id and no foreign-id anymore. This opens up
+bulk-processing by looking at the whole table and combining
+flag-setting to reduce imap-roundtrips and select-folder calls.
+
+Concretely, these IMAP jobs:
+
+    DeleteMsgOnImap
+    MarkseenMsgOnImap
+    MoveMsg
+
+Would be replaced by a few per-message columns in the new msgwork table:
+
+- needs_mark_seen: (bool) message shall be marked as seen on imap
+- needs_to_move: (bool) message should be moved to mvbox_folder
+- deletion_time: (target_time or 0) message shall be deleted at specified time
+- needs_send_mdn: (bool) MDN shall be sent
+
+The various places that currently add the (replaced) jobs
+would now add/modify the respective message record in the message-work table.
+
+Looking at a single message-work entry conceptually looks like this::
+
+    if msg.server_uid==0:
+        return RetryLater  # nothing can be done without server_uid
+
+    if msg.deletion_time > current_time:
+        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
+        clear(needs_deletion)
+        clear(mark_seen)
+
+    if needs_mark_seen:
+        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
+        clear(needs_mark_seen)
+
+    if needs_send_mdn:
+        schedule_smtp_send_mdn(msg)
+        clear(needs_send_mdn)
+
+    if any_flag_set():
+        retrylater
+    # remove msgwork entry from table
+
+
+Notes/Questions:
+
+- it's unclear how much we need per-message retry-time tracking/backoff
+
+- drafting bulk processing algo is useful before
+  going for the implementation, i.e. including select_folder calls etc.
+
+- maybe it's better to not have bools for the flags but
+
+  0 (no change)
+  1 (set the imap flag)
+  2 (clear the imap flag)
+
+  and design such that we can cover all imap flags.
+
+- It might not be neccessary to keep needs_send_mdn state in this table
+  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/cmdline.rs

@@ -2,7 +2,7 @@ use std::str::FromStr;
 
 use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
+use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -183,7 +183,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -202,6 +202,15 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.get_viewtype() == Viewtype::VideochatInvitation {
+            format!(
+                "[VIDEOCHAT-INVITATION: {}, type={}]",
+                msg.get_videochat_url().unwrap_or_default(),
+                msg.get_videochat_type().unwrap_or_default()
+            )
+        } else {
+            "".to_string()
+        },
         if msg.is_forwarded() {
             "[FORWARDED]"
         } else {
@@ -215,7 +224,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
 async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id.is_daymarker() {
+        if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
             println!(
                 "--------------------------------------------------------------------------------"
             );
@@ -359,6 +368,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
                  listmedia\n\
@@ -488,6 +498,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
+            let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
                 &context,
                 listflags,
@@ -495,6 +506,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
+            let time_needed = std::time::SystemTime::now()
+                .duration_since(time_start)
+                .unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -553,6 +567,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
+            println!("{:?} to create this list", time_needed);
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
@@ -569,6 +584,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
+            let msglist: Vec<MsgId> = msglist
+                .into_iter()
+                .map(|x| match x {
+                    ChatItem::Message { msg_id } => msg_id,
+                    ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
+                    ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
+                })
+                .collect();
+
             let members = chat::get_chat_contacts(&context, sel_chat.id).await;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
@@ -794,6 +818,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "videochat" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
+        }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 

examples/repl/main.rs

@@ -20,6 +20,7 @@ use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
 use deltachat::Event;
+use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
@@ -35,34 +36,34 @@ use self::cmdline::*;
 /// Event Handler
 fn receive_event(event: Event) {
     let yellow = Color::Yellow.normal();
-    let red = Color::Red.normal();
     match event {
         Event::Info(msg) => {
-            println!("[INFO] {}", msg);
+            /* do not show the event as this would fill the screen */
+            info!("{}", msg);
         }
         Event::SmtpConnected(msg) => {
-            println!("[INFO SMTP_CONNECTED] {}", msg);
+            info!("[SMTP_CONNECTED] {}", msg);
         }
         Event::ImapConnected(msg) => {
-            println!("[INFO IMAP_CONNECTED] {}", msg);
+            info!("[IMAP_CONNECTED] {}", msg);
         }
         Event::SmtpMessageSent(msg) => {
-            println!("[INFO SMTP_MESSAGE_SENT] {}", msg);
+            info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
         Event::Warning(msg) => {
-            println!("[WARNING] {}", msg);
+            warn!("{}", msg);
         }
         Event::Error(msg) => {
-            println!("[ERROR] {}", red.paint(msg));
+            error!("{}", msg);
         }
         Event::ErrorNetwork(msg) => {
-            println!("[ERROR NETWORK] msg={}", red.paint(msg));
+            error!("[NETWORK] msg={}", msg);
         }
         Event::ErrorSelfNotInGroup(msg) => {
-            println!("[ERROR SELF_NOT_IN_GROUP] {}", red.paint(msg));
+            error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
         Event::MsgsChanged { chat_id, msg_id } => {
-            println!(
+            info!(
                 "{}",
                 yellow.paint(format!(
                     "Received MSGS_CHANGED(chat_id={}, msg_id={})",
@@ -71,40 +72,40 @@ fn receive_event(event: Event) {
             );
         }
         Event::ContactsChanged(_) => {
-            println!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
+            info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
         Event::LocationChanged(contact) => {
-            println!(
+            info!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
         Event::ConfigureProgress(progress) => {
-            println!(
+            info!(
                 "{}",
                 yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
             );
         }
         Event::ImexProgress(progress) => {
-            println!(
+            info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
         Event::ImexFileWritten(file) => {
-            println!(
+            info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
         Event::ChatModified(chat) => {
-            println!(
+            info!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
             );
         }
         _ => {
-            println!("Received {}", yellow.paint(format!("{:?}", event)));
+            info!("Received {:?}", event);
         }
     }
 }
@@ -157,7 +158,7 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 27] = [
     "listchats",
     "listarchived",
     "chat",
@@ -177,6 +178,7 @@ const CHAT_COMMANDS: [&str; 26] = [
     "send",
     "sendimage",
     "sendfile",
+    "videochat",
     "draft",
     "listmedia",
     "archive",

python/CHANGELOG

@@ -1,6 +1,17 @@
-0.900.0 (DRAFT)
+1.40.1
 ---------------
 
+- emit "ac_member_removed" event (with 'actor' being the removed contact) 
+  for when a user leaves a group.
+
+- fix create_contact(addr) when addr is the self-contact. 
+
+
+1.40.0
+---------------
+
+- uses latest 1.40+ Delta Chat core
+
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -10,6 +21,7 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
+
 0.800.0
 -------
 

python/README.rst

@@ -7,76 +7,14 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
-Installing bindings from source (Updated: 20-Jan-2020)
-=========================================================
-
-Install Rust and Cargo first.  Deltachat needs a specific nightly
-version, the easiest is probably to first install Rust stable from
-rustup and then use this to install the correct nightly version.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then GIT clone the deltachat-core-rust repo and get the actual
-rust- and cargo-toolchain needed by deltachat::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-   rustup show
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   ./install_python_bindings.py
-
-The installation might take a while, depending on your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_NEW_TMP_EMAIL`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs. This is implemented on the server side via
-  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
+without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
+yourself <sourceinstall>`_.
 
-We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -103,6 +41,78 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
+Running tests
+=============
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+This will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real e-mail servers.
+
+.. _livetests:
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
+
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
+
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
+One hour is enough to invoke pytest and run all offline and online tests:
+
+    pytest
+
+    # or if you have installed pytest-xdist for parallel test execution
+    pytest -n6
+
+Each test run creates new accounts.
+
+
+.. _sourceinstall:
+
+Installing bindings from source (Updated: July 2020)
+=========================================================
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   python install_python_bindings.py
+
+The core compilation and bindings building might take a while,
+depending on the speed of your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+
 Code examples
 =============
 

python/doc/_templates/globaltoc.html

@@ -9,8 +9,7 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
-  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
+  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/echo_and_quit.py

@@ -12,7 +12,7 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
                 message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))
@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
+    def ac_member_added(self, chat, contact, actor, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
+    def ac_member_removed(self, chat, contact, actor, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -26,15 +26,15 @@ def test_echo_quit_plugin(acfactory, lp):
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
+    bot_chat = bot_contact.create_chat()
+    bot_chat.send_text("hello")
 
-    lp.sec("waiting for the bot-reply to arrive")
+    lp.sec("waiting for the reply message from the bot to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    assert reply.chat == ch1
     lp.sec("send quit sequence")
-    ch1.send_text("/quit")
+    bot_chat.send_text("/quit")
     botproc.wait()
 
 
@@ -47,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+    ac1.add_account_plugin(FFIEventLogger(ac1))
+    ac2.add_account_plugin(FFIEventLogger(ac2))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
+        *ac_member_added {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))
+        *ac_member_removed {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -60,7 +60,8 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = events.FFIEventLogger(ac, "bot")
+        ac.set_config("displayname", "bot")
+        log = events.FFIEventLogger(ac)
         ac.add_account_plugin(log)
 
     for plugin in account_plugins or []:
@@ -76,8 +77,8 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.configure()
-        ac.wait_configure_finish()
+        configtracker = ac.configure()
+        configtracker.wait_finish()
 
     # start IO threads and configure if neccessary
     ac.start_io()

python/src/deltachat/_build.py

@@ -1,65 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -71,13 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -213,22 +213,39 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, email, name=None):
-        """ create a (new) Contact. If there already is a Contact
-        with that e-mail address, it is unblocked and its name is
-        updated.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        :param email: email-address (text type)
-        :param name: display name for this contact (optional)
+        Calling this method will always resulut in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        `name` is updated if specified.
+
+        :param obj: email-address, Account or Contact instance.
+        :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
+        if isinstance(obj, Account):
+            if not obj.is_configured():
+                raise ValueError("can only add addresses from configured accounts")
+            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
+        elif isinstance(obj, Contact):
+            if obj.account != self:
+                raise ValueError("account mismatch {}".format(obj))
+            addr, displayname = obj.addr, obj.name
+        elif isinstance(obj, str):
+            displayname, addr = parseaddr(obj)
+        else:
+            raise TypeError("don't know how to create chat for %r" % (obj, ))
+
+        if name is None and displayname:
+            name = displayname
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -250,6 +267,13 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -279,53 +303,29 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+    def create_chat(self, obj):
+        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        return self.create_contact(obj).create_chat()
 
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(contact, "id"):
-            if contact.account != self:
-                raise ValueError("Contact belongs to a different Account")
-            contact_id = contact.id
-        else:
-            assert isinstance(contact, int)
-            contact_id = contact
-        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
-        return Chat(self, chat_id)
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
 
-    def create_chat_by_message(self, message):
-        """ create or get an existing chat object for the
-        the specified message.
-
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
-        :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(message, "id"):
-            if message.account != self:
-                raise ValueError("Message belongs to a different Account")
-            msg_id = message.id
-        else:
-            assert isinstance(message, int)
-            msg_id = message
-        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
-        return Chat(self, chat_id)
-
-    def create_group_chat(self, name, verified=False):
+    def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
+        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        return Chat(self, chat_id)
+        chat = Chat(self, chat_id)
+        if contacts is not None:
+            for contact in contacts:
+                chat.add_contact(contact)
+        return chat
 
     def get_chats(self):
         """ return list of chats.
@@ -354,13 +354,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -567,29 +560,24 @@ class Account(object):
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
-        :returns: None (account is configured and with io-scheduling running)
+        :returns: None
         """
         if not self.is_configured():
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
     def configure(self):
+        """ Start configuration process and return a Configtracker instance
+        on which you can block with wait_finish() to get a True/False success
+        value for the configuration process.
+        """
         assert not self.is_configured()
-        assert not hasattr(self, "_configtracker")
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
-        if hasattr(self, "_configtracker"):
-            self.remove_account_plugin(self._configtracker)
-        self._configtracker = ConfigureTracker()
-        self.add_account_plugin(self._configtracker)
+        configtracker = ConfigureTracker(self)
+        self.add_account_plugin(configtracker)
         lib.dc_configure(self._dc_context)
-
-    def wait_configure_finish(self):
-        try:
-            self._configtracker.wait_finish()
-        finally:
-            self.remove_account_plugin(self._configtracker)
-            del self._configtracker
+        return configtracker
 
     def is_started(self):
         return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
@@ -618,12 +606,24 @@ class Account(object):
         self.stop_io()
 
         self.log("remove dc_context references")
-        # the dc_context_unref triggers get_next_event to return ffi.NULL
-        # which in turns makes the event thread finish execution
+
+        # if _dc_context is unref'ed the event thread should quickly
+        # receive the termination signal. However, some python code might
+        # still hold a reference and so we use a secondary signal
+        # to make sure the even thread terminates if it receives any new
+        # event, indepedently from waiting for the core to send NULL to
+        # get_next_event().
+        self._event_thread.mark_shutdown()
         self._dc_context = None
 
         self.log("wait for event thread to finish")
-        self._event_thread.wait()
+        try:
+            self._event_thread.wait(timeout=2)
+        except RuntimeError as e:
+            self.log("Waiting for event thread failed: {}".format(e))
+
+        if self._event_thread.is_alive():
+            self.log("WARN: event thread did not terminate yet, ignoring.")
 
         self._shutdown_event.set()
 

python/src/deltachat/chat.py

@@ -18,6 +18,8 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -137,6 +139,22 @@ class Chat(object):
         """
         return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
+    def get_ephemeral_timer(self):
+        """ get ephemeral timer.
+
+        :returns: ephemeral timer value in seconds
+        """
+        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
+
+    def set_ephemeral_timer(self, timer):
+        """ set ephemeral timer.
+
+        :param: timer value in seconds
+
+        :returns: None
+        """
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
+
     def get_type(self):
         """ (deprecated) return type of this chat.
 
@@ -328,33 +346,34 @@ class Chat(object):
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, contact):
+    def add_contact(self, obj):
         """ add a contact to this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
+        return contact
 
-    def remove_contact(self, contact):
+    def remove_contact(self, obj):
         """ remove a contact from this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
-        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-
         """
         from .contact import Contact
         dc_array = ffi.gc(
@@ -365,6 +384,14 @@ class Chat(object):
             dc_array, lambda id: Contact(self.account, id))
         )
 
+    def num_contacts(self):
+        """ return number of contacts in this chat. """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return lib.dc_array_get_cnt(dc_array)
+
     def set_profile_image(self, img_path):
         """Set group profile image.
 

python/src/deltachat/const.py

@@ -1,197 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -3,6 +3,8 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
+from .chat import Chat
+from . import const
 
 
 class Contact(object):
@@ -11,6 +13,8 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -36,10 +40,13 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def display_name(self):
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
+    # deprecated alias
+    display_name = name
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -58,6 +65,16 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)
+    def create_chat(self):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        dc_context = self.account._dc_context
+        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
+        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
+        return Chat(self.account, chat_id)
+
+    # deprecated name
+    get_chat = create_chat

python/src/deltachat/direct_imap.py

@@ -0,0 +1,218 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
+import io
+import email
+import ssl
+import pathlib
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
+
+
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+ALL = "1:*"
+
+
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account):
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        ssl_context = ssl.create_default_context()
+
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
+        try:
+            self.conn.logout()
+        except (OSError, IMAPClientError):
+            print("Could not logout direct_imap conn")
+
+    def select_folder(self, foldername):
+        assert not self._idling
+        return self.conn.select_folder(foldername)
+
+    def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
+        if "_" not in config_name:
+            config_name = "configured_{}_folder".format(config_name)
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+
+    def list_folders(self):
+        """ return list of all existing folder names"""
+        assert not self._idling
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
+
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self):
+        assert not self._idling
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
+        assert not self._idling
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.set_flags(messages, [SEEN])
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self):
+        return len(self.get_unread_messages())
+
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            requested = [b'BODY.PEEK[]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
+                    continue
+
+                flags = data[FLAGS]
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        self.account.log("imap-direct: idle_check returned {!r}".format(res))
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/events.py

@@ -28,13 +28,9 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
+    def __init__(self, account):
         self.account = account
-        self.logid = logid
+        self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
@@ -90,11 +86,11 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
+    def get_info_contains(self, regex):
+        rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
+            if rex.search(ev.data2):
                 return ev
 
     def ensure_event_not_queued(self, event_name_regex):
@@ -127,6 +123,12 @@ class FFIEventTracker:
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
 
+    def wait_msg_delivered(self, msg):
+        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == msg.chat.id
+        assert ev.data2 == msg.id
+        assert msg.is_out_delivered()
+
 
 class EventThread(threading.Thread):
     """ Event Thread for an account.
@@ -137,6 +139,7 @@ class EventThread(threading.Thread):
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
+        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -145,12 +148,15 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def wait(self):
+    def mark_shutdown(self):
+        self._marked_for_shutdown = True
+
+    def wait(self, timeout=None):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join()
+        self.join(timeout=timeout)
 
     def run(self):
         """ get and run events until shutdown. """
@@ -162,10 +168,12 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while 1:
+        while not self._marked_for_shutdown:
             event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
+            if self._marked_for_shutdown:
+                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are not allowed to block/last long as this
+    Hooks are generally not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,10 +31,6 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -43,7 +39,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -55,19 +51,37 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
+        """ Called when an outgoing message has been delivered to SMTP.
+
+        :param message: Message that was just delivered.
+        """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
+        """ Chat was created or modified regarding membership, avatar, title.
+
+        :param chat: Chat which was modified.
+        """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
+    def ac_member_added(self, chat, contact, actor, message):
+        """ Called for each contact added to an accepted chat.
+
+        :param chat: Chat where contact was added.
+        :param contact: Contact that was added.
+        :param actor: Who added the contact (None if it was our self-addr)
+        :param message: The original system message that reports the addition.
+        """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
+    def ac_member_removed(self, chat, contact, actor, message):
+        """ Called for each contact removed from a chat.
+
+        :param chat: Chat where contact was removed.
+        :param contact: Contact that was removed.
+        :param actor: Who removed the contact (None if it was our self-addr)
+        :param message: The original system message that reports the removal.
+        """
 
 
 class Global:
@@ -88,6 +102,14 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
+    @global_hookspec
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -1,6 +1,7 @@
 """ The Message object. """
 
 import os
+import re
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -53,15 +54,19 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
+    def create_chat(self):
+        """ create or get an existing chat (group) object for this message.
+
+        If the message is a deaddrop contact request
+        the sender will become an accepted contact.
+
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
-                lib.dc_msg_unref
-        )
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):
@@ -150,6 +155,26 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @props.with_doc
+    def ephemeral_timer(self):
+        """Ephemeral timer in seconds
+
+        :returns: timer in seconds or None if there is no timer
+        """
+        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
+        if timer:
+            return timer
+
+    @props.with_doc
+    def ephemeral_timestamp(self):
+        """UTC time when the message will be deleted.
+
+        :returns: naive datetime.datetime() object or None if the timer is not started.
+        """
+        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
+        if ts:
+            return datetime.utcfromtimestamp(ts)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -332,20 +357,43 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
+        if not res:
+            return
+        action, affected, actor = res
+        affected = msg.account.get_contact_by_addr(affected)
+        if actor == "me":
+            actor = None
+        else:
+            actor = msg.account.get_contact_by_addr(actor)
+        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
+        return "ac_member_" + res[0], d
+
+
+def extract_addr(text):
+    m = re.match(r'.*\((.+@.+)\)', text)
+    if m:
+        text = m.group(1)
+    text = text.rstrip(".")
+    return text.strip()
 
 
 def parse_system_add_remove(text):
+    """ return add/remove info from parsing the given system message text.
+
+    returns a (action, affected, actor) triple """
+
     # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
+    # Member x@y added by a@b
+    # Member With space (tmp1@x.org) removed by tmp2@x.org.
+    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+    # Group left by some one (tmp1@x.org).
+    # Group left by tmp1@x.org.
     text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")
+    m = re.match(r'member (.+) (removed|added) by (.+)', text)
+    if m:
+        affected, action, actor = m.groups()
+        return action, extract_addr(affected), extract_addr(actor)
+    if text.startswith("group left by "):
+        addr = extract_addr(text[13:])
+        if addr:
+            return "removed", addr, addr

python/src/deltachat/testplugin.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import sys
+import io
 import subprocess
 import queue
 import threading
@@ -16,6 +17,7 @@ from . import Account, const
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
@@ -30,12 +32,13 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
+    parser.addoption(
+        "--strict-tls", action="store_true",
+        help="Never accept invalid TLS certificates for test accounts",
+    )
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
@@ -153,7 +156,7 @@ class SessionLiveConfigFromURL:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
+                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
@@ -216,6 +219,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -226,13 +230,18 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+                logger = FFIEventLogger(ac)
+                logger.init_time = self.init_time
+                ac.add_account_plugin(logger)
             self._accounts.append(ac)
             return ac
 
@@ -242,10 +251,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
+            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -280,16 +286,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            if pytestconfig.getoption("--strict-tls"):
+                # Enable strict certificate checks for online accounts
+                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
@@ -301,31 +306,27 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1.wait_configure_finish()
-            ac1.start_io()
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1.wait_configure_finish()
-            ac1.start_io()
-            ac2.wait_configure_finish()
-            ac2.start_io()
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True, quiet=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
+            self.wait_configure_and_start_io()
             for acc in accounts:
-                acc._configtracker.wait_finish()
-                acc.start_io()
+                acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
@@ -334,8 +335,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
@@ -343,14 +342,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
                 "-u",
@@ -375,9 +389,42 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    return am
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
@@ -445,4 +492,17 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
+
     return Printer()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # set a report attribute for each phase of a call, which can
+    # be "setup", "call", "teardown"
+
+    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -40,12 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -59,7 +61,10 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
+        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """

python/tests/stress_test_db.py

@@ -110,7 +110,7 @@ class AutoReplier:
         if self.current_sent >= self.num_send:
             self.report_func(self, ReportType.exit)
             return
-        message.accept_sender_contact()
+        message.create_chat()
         message.mark_seen()
         self.log("incoming message: {}".format(message))
 

python/tests/test_increation.py

@@ -36,9 +36,7 @@ def wait_msgs_changed(account, msgs_list):
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -48,9 +46,7 @@ class TestOnlineInCreation:
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -64,9 +60,7 @@ class TestOnlineInCreation:
     def test_forward_increation(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
@@ -80,7 +74,7 @@ class TestOnlineInCreation:
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(c2)
+        chat2.add_contact(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them

python/tests/test_lowlevel.py

@@ -69,8 +69,8 @@ def test_sig():
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
 
-    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    chat = ac1.create_chat_by_contact(contact1)
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]

python/tox.ini

@@ -7,7 +7,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
@@ -71,6 +71,8 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
+markers = 
+    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

rust-toolchain

@@ -1 +1 @@
-1.43.1
+1.45.0

set_core_version.py

@@ -5,19 +5,29 @@ import sys
 import re
 import pathlib
 import subprocess
+from argparse import ArgumentParser
 
 rex = re.compile(r'version = "(\S+)"')
 
-def read_toml_version(relpath):
+
+def regex_matches(relpath, regex=rex):
     p = pathlib.Path(relpath)
     assert p.exists()
     for line in open(str(p)):
-        m = rex.match(line)
+        m = regex.match(line)
         if m is not None:
-            return m.group(1)
+            return m
+
+
+def read_toml_version(relpath):
+    res = regex_matches(relpath, rex)
+    if res is not None:
+        return res.group(1)
     raise ValueError("no version found in {}".format(relpath))
 
-def replace_toml_version(relpath, newversion):
+
+def replace_toml_version_and_lto(relpath, newversion):
+    lto_rex = re.compile(r'#?\s*lto =.*')
     p = pathlib.Path(relpath)
     assert p.exists()
     tmp_path = str(p) + "_tmp"
@@ -25,18 +35,32 @@ def replace_toml_version(relpath, newversion):
         for line in open(str(p)):
             m = rex.match(line)
             if m is not None:
+                print("{}: set version={}".format(relpath, newversion))
                 f.write('version = "{}"\n'.format(newversion))
             else:
+                m = lto_rex.match(line)
+                if m:
+                    print("{}: setting lto = true".format(relpath))
+                    line = "lto = true\n"
                 f.write(line)
     os.rename(tmp_path, str(p))
 
-if __name__ == "__main__":
 
-    if len(sys.argv) < 2:
-        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
+def main():
+    parser = ArgumentParser(prog="set_core_version")
+    parser.add_argument("newversion")
+
+    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml"]
+    try:
+        opts = parser.parse_args()
+    except SystemExit:
+        print()
+        for x in toml_list:
             print("{}: {}".format(x, read_toml_version(x)))
+        print()
         raise SystemExit("need argument: new version, example: 1.25.0")
-    newversion = sys.argv[1]
+
+    newversion = opts.newversion
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
 
@@ -52,10 +76,13 @@ if __name__ == "__main__":
     else:
         raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
 
-    replace_toml_version("Cargo.toml", newversion)
-    replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
+    replace_toml_version_and_lto("Cargo.toml", newversion)
+    replace_toml_version_and_lto("deltachat-ffi/Cargo.toml", newversion)
 
+    print("running cargo check")
     subprocess.call(["cargo", "check"])
+
+    print("adding changes to git index")
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
@@ -63,3 +90,8 @@ if __name__ == "__main__":
     print("")
     print("   git tag {}".format(newversion))
     print("")
+
+
+if __name__ == "__main__":
+    main()
+

spec.md

@@ -1,10 +1,12 @@
-# Chat-over-Email specification
+# chat-mail specification
 
-Version 0.30.0
+Version: 0.32.0
+Status:  In-progress 
+Format:  [Semantic Line Breaks](https://sembr.org/)
 
-This document describes how emails can be used
-to implement typical messenger functions
-while staying compatible to existing MUAs.
+This document roughly describes how chat-mail 
+apps use the standard e-mail system 
+to implement typical messenger functions.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -30,17 +32,14 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
-If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD start with the characters `Chat:`
-and SHOULD be an excerpt of the message.
+the `Subject` header SHOULD be `Message from <sender name>`.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -58,7 +57,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Chat: Hello ...
+    Subject: Message from sender@domain
 
     Hello world!
 

src/blob.rs

@@ -8,11 +8,15 @@ use async_std::prelude::*;
 use async_std::{fs, io};
 
 use image::GenericImageView;
+use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::constants::AVATAR_SIZE;
+use crate::config::Config;
+use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::Event;
+use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -57,7 +61,7 @@ impl<'a> BlobObject<'a> {
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -164,6 +168,9 @@ impl<'a> BlobObject<'a> {
     /// subdirectory is used and [BlobObject::sanitise_name] does not
     /// modify the filename.
     ///
+    /// Paths into the blob directory may be either defined by an absolute path
+    /// or by the relative prefix `$BLOBDIR`.
+    ///
     /// # Errors
     ///
     /// This merely delegates to the [BlobObject::create_and_copy] and
@@ -175,6 +182,11 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'_>, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
+        } else if src.as_ref().starts_with("$BLOBDIR/") {
+            BlobObject::from_name(
+                context,
+                src.as_ref().to_str().unwrap_or_default().to_string(),
+            )
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -372,11 +384,73 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
     }
+
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
+            != Some((Viewtype::Image, "image/jpeg"))
+        {
+            return Ok(());
+        }
+
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
+
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
+        }
+
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
+    }
+
+    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exifreader = exif::Reader::new();
+        let exif = exifreader.read_from_container(&mut bufreader)?;
+        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
+            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
+            // we only use rotation, in practise, flipping is not used.
+            match orientation.value.get_uint(0) {
+                Some(3) => return Ok(180),
+                Some(6) => return Ok(90),
+                Some(8) => return Ok(270),
+                other => warn!(context, "exif orientation value ignored: {:?}", other),
+            }
+        }
+        Ok(0)
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -400,7 +474,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
@@ -431,7 +505,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
         let fname = t.ctx.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
@@ -442,7 +516,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_lowercase_ext() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
             .await
             .unwrap();
@@ -451,7 +525,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_file_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -460,7 +534,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_rel_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -469,7 +543,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_suffix() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -480,7 +554,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_dup() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -504,7 +578,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_double_ext_preserved() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
             .await
             .unwrap();
@@ -529,7 +603,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_long_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let s = "1".repeat(150);
         let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
@@ -538,7 +612,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_and_copy() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
         let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
@@ -554,7 +628,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_from_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
@@ -572,7 +646,7 @@ mod tests {
     }
     #[async_std::test]
     async fn test_create_from_name_long() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
         let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
@@ -594,8 +668,40 @@ mod tests {
 
     #[test]
     fn test_sanitise_name() {
-        let (_, ext) =
+        let (stem, ext) =
             BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
         assert_eq!(ext, ".txt");
+        assert!(!stem.is_empty());
+
+        // the extensions are kept together as between stem and extension a number may be added -
+        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
+        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
+        assert_eq!(stem, "wot");
+        assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
+        assert_eq!(stem, "");
+        assert_eq!(ext, ".foo.bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
+        assert!(stem.contains("foo"));
+        assert!(!stem.contains("?"));
+        assert_eq!(ext, ".bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("no-extension");
+        assert_eq!(stem, "no-extension");
+        assert_eq!(ext, "");
+
+        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
+        assert_eq!(ext, ".c");
+        assert!(!stem.contains("path"));
+        assert!(!stem.contains("ignored"));
+        assert!(stem.contains("this"));
+        assert!(stem.contains("forbidden"));
+        assert!(!stem.contains("/"));
+        assert!(!stem.contains("\\"));
+        assert!(!stem.contains(":"));
+        assert!(!stem.contains("*"));
+        assert!(!stem.contains("?"));
     }
 }

src/chat.rs

@@ -15,6 +15,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::{delete_expired_messages, schedule_ephemeral_task, Timer as EphemeralTimer};
 use crate::error::{bail, ensure, format_err, Error};
 use crate::events::Event;
 use crate::job::{self, Action};
@@ -24,6 +25,25 @@ use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
+/// An chat item, such as a message or a marker.
+#[derive(Debug, Copy, Clone)]
+pub enum ChatItem {
+    Message {
+        msg_id: MsgId,
+    },
+
+    /// A marker without inherent meaning. It is inserted before user
+    /// supplied MsgId.
+    Marker1,
+
+    /// Day marker, separating messages that correspond to different
+    /// days according to local time.
+    DayMarker {
+        /// Marker timestamp, for day markers
+        timestamp: i64,
+    },
+}
+
 /// Chat ID, including reserved IDs.
 ///
 /// Some chat IDs are reserved to identify special chat types.  This
@@ -39,18 +59,9 @@ impl ChatId {
         ChatId(id)
     }
 
-    /// A ChatID which indicates an error.
-    ///
-    /// This is transitional and should not be used in new code.  Do
-    /// not represent errors in a ChatId.
-    pub fn is_error(self) -> bool {
-        self.0 == 0
-    }
-
     /// An unset ChatId
     ///
-    /// Like [ChatId::is_error], from which it is indistinguishable, this is
-    /// transitional and should not be used in new code.
+    /// This is transitional and should not be used in new code.
     pub fn is_unset(self) -> bool {
         self.0 == 0
     }
@@ -431,24 +442,44 @@ impl ChatId {
     }
 
     async fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
-        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
-        self.parent_query(
-            context,
-            "rfc724_mid, mime_in_reply_to, mime_references",
-            collect,
-        )
-        .await
-        .ok()
-        .flatten()
+        let collect =
+            |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?));
+        let (rfc724_mid, mime_in_reply_to, mime_references, error): (
+            String,
+            String,
+            String,
+            String,
+        ) = self
+            .parent_query(
+                context,
+                "rfc724_mid, mime_in_reply_to, mime_references, error",
+                collect,
+            )
+            .await
+            .ok()
+            .flatten()?;
+
+        if !error.is_empty() {
+            // Do not reply to error messages.
+            //
+            // An error message could be a group chat message that we failed to decrypt and
+            // assigned to 1:1 chat. A reply to it will show up as a reply to group message
+            // on the other side. To avoid such situations, it is better not to reply to
+            // error messages at all.
+            None
+        } else {
+            Some((rfc724_mid, mime_in_reply_to, mime_references))
+        }
     }
 
     async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
-        let collect = |row: &rusqlite::Row| Ok(row.get(0)?);
-        let packed: Option<String> = self.parent_query(context, "param", collect).await?;
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?));
+        let res: Option<(String, String)> =
+            self.parent_query(context, "param, error", collect).await?;
 
-        if let Some(ref packed) = packed {
+        if let Some((ref packed, ref error)) = res {
             let param = packed.parse::<Params>()?;
-            Ok(param.exists(Param::GuaranteeE2ee))
+            Ok(error.is_empty() && param.exists(Param::GuaranteeE2ee))
         } else {
             // No messages
             Ok(false)
@@ -699,6 +730,7 @@ impl Chat {
                 .unwrap_or_else(std::path::PathBuf::new),
             draft,
             is_muted: self.is_muted(),
+            ephemeral_timer: self.id.get_ephemeral_timer(context).await?,
         })
     }
 
@@ -875,11 +907,10 @@ impl Chat {
 
                     // the whole list of messages referenced may be huge;
                     // only use the oldest and and the parent message
-                    let parent_references = if let Some(n) = parent_references.find(' ') {
-                        &parent_references[0..n]
-                    } else {
-                        &parent_references
-                    };
+                    let parent_references = parent_references
+                        .find(' ')
+                        .and_then(|n| parent_references.get(..n))
+                        .unwrap_or(&parent_references);
 
                     if !parent_references.is_empty() && !parent_rfc724_mid.is_empty() {
                         // angle brackets are added by the mimefactory later
@@ -927,10 +958,20 @@ impl Chat {
                     .await?;
             }
 
+            let ephemeral_timer = if msg.param.get_cmd() == SystemMessage::EphemeralTimerChanged {
+                EphemeralTimer::Disabled
+            } else {
+                self.id.get_ephemeral_timer(context).await?
+            };
+            let ephemeral_timestamp = match ephemeral_timer {
+                EphemeralTimer::Disabled => 0,
+                EphemeralTimer::Enabled { duration } => timestamp + i64::from(duration),
+            };
+
             // add message to the database
 
             if context.sql.execute(
-                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?);",
+                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id, ephemeral_timer, ephemeral_timestamp) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?);",
                         paramsv![
                             new_rfc724_mid,
                             self.id,
@@ -945,6 +986,8 @@ impl Chat {
                             new_in_reply_to,
                             new_references,
                             location_id as i32,
+                            ephemeral_timer,
+                            ephemeral_timestamp
                         ]
                     ).await.is_ok() {
                         msg_id = context.sql.get_rowid(
@@ -963,6 +1006,7 @@ impl Chat {
         } else {
             error!(context, "Cannot send message, not configured.",);
         }
+        schedule_ephemeral_task(context).await;
 
         Ok(MsgId::new(msg_id))
     }
@@ -990,13 +1034,13 @@ impl rusqlite::types::ToSql for ChatVisibility {
 
 impl rusqlite::types::FromSql for ChatVisibility {
     fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-        i64::column_result(value).and_then(|val| {
+        i64::column_result(value).map(|val| {
             match val {
-                2 => Ok(ChatVisibility::Pinned),
-                1 => Ok(ChatVisibility::Archived),
-                0 => Ok(ChatVisibility::Normal),
+                2 => ChatVisibility::Pinned,
+                1 => ChatVisibility::Archived,
+                0 => ChatVisibility::Normal,
                 // fallback to to Normal for unknown values, may happen eg. on imports created by a newer version.
-                _ => Ok(ChatVisibility::Normal),
+                _ => ChatVisibility::Normal,
             }
         })
     }
@@ -1059,6 +1103,9 @@ pub struct ChatInfo {
     ///
     /// The exact time its muted can be found out via the `chat.mute_duration` property
     pub is_muted: bool,
+
+    /// Ephemeral message timer.
+    pub ephemeral_timer: EphemeralTimer,
     // ToDo:
     // - [ ] deaddrop,
     // - [ ] summary,
@@ -1327,11 +1374,12 @@ pub(crate) fn msgtype_has_file(msgtype: Viewtype) -> bool {
         Viewtype::Voice => true,
         Viewtype::Video => true,
         Viewtype::File => true,
+        Viewtype::VideochatInvitation => false,
     }
 }
 
 async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Error> {
-    if msg.viewtype == Viewtype::Text {
+    if msg.viewtype == Viewtype::Text || msg.viewtype == Viewtype::VideochatInvitation {
         // the caller should check if the message text is empty
     } else if msgtype_has_file(msg.viewtype) {
         let blob = msg
@@ -1341,6 +1389,12 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Er
             .ok_or_else(|| {
                 format_err!("Attachment missing for message of type #{}", msg.viewtype)
             })?;
+
+        if msg.viewtype == Viewtype::Image {
+            if let Err(e) = blob.recode_to_image_size(context).await {
+                warn!(context, "Cannot recode image, using original data: {:?}", e);
+            }
+        }
         msg.param.set(Param::File, blob.as_name());
 
         if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
@@ -1447,7 +1501,7 @@ pub async fn send_msg(
                 }
             }
             msg.param.remove(Param::PrepForwards);
-            msg.save_param_to_disk(context).await;
+            msg.update_param(context).await;
         }
         return send_msg_inner(context, chat_id, msg).await;
     }
@@ -1558,16 +1612,59 @@ pub async fn send_text_msg(
     send_msg(context, chat_id, &mut msg).await
 }
 
+pub async fn send_videochat_invitation(context: &Context, chat_id: ChatId) -> Result<MsgId, Error> {
+    ensure!(
+        !chat_id.is_special(),
+        "video chat invitation cannot be sent to special chat: {}",
+        chat_id
+    );
+
+    let instance = if let Some(instance) = context.get_config(Config::WebrtcInstance).await {
+        if !instance.is_empty() {
+            instance
+        } else {
+            bail!("webrtc_instance is empty");
+        }
+    } else {
+        bail!("webrtc_instance not set");
+    };
+
+    let room = dc_create_id();
+
+    let instance = if instance.contains("$ROOM") {
+        instance.replace("$ROOM", &room)
+    } else {
+        format!("{}{}", instance, room)
+    };
+
+    let mut msg = Message::new(Viewtype::VideochatInvitation);
+    msg.param.set(Param::WebrtcRoom, &instance);
+    msg.text = Some(
+        context
+            .stock_string_repl_str(
+                StockMessage::VideochatInviteMsgBody,
+                Message::parse_webrtc_instance(&instance).1,
+            )
+            .await,
+    );
+    send_msg(context, chat_id, &mut msg).await
+}
+
 pub async fn get_chat_msgs(
     context: &Context,
     chat_id: ChatId,
     flags: u32,
     marker1before: Option<MsgId>,
-) -> Vec<MsgId> {
-    match delete_device_expired_messages(context).await {
+) -> Vec<ChatItem> {
+    match delete_expired_messages(context).await {
         Err(err) => warn!(context, "Failed to delete expired messages: {}", err),
         Ok(messages_deleted) => {
             if messages_deleted {
+                // Trigger reload of chatlist.
+                //
+                // On desktop chatlist is always shown on the side,
+                // and it is important to update the last message shown
+                // there.
                 context.emit_event(Event::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
@@ -1586,18 +1683,20 @@ pub async fn get_chat_msgs(
             let (curr_id, ts) = row?;
             if let Some(marker_id) = marker1before {
                 if curr_id == marker_id {
-                    ret.push(MsgId::new(DC_MSG_ID_MARKER1));
+                    ret.push(ChatItem::Marker1);
                 }
             }
             if (flags & DC_GCM_ADDDAYMARKER) != 0 {
                 let curr_local_timestamp = ts + cnv_to_local;
                 let curr_day = curr_local_timestamp / 86400;
                 if curr_day != last_day {
-                    ret.push(MsgId::new(DC_MSG_ID_DAYMARKER));
+                    ret.push(ChatItem::DayMarker {
+                        timestamp: curr_day,
+                    });
                     last_day = curr_day;
                 }
             }
-            ret.push(curr_id);
+            ret.push(ChatItem::Message { msg_id: curr_id });
         }
         Ok(ret)
     };
@@ -1729,52 +1828,6 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-/// Deletes messages which are expired according to "delete_device_after" setting.
-///
-/// Returns true if any message is deleted, so event can be emitted. If nothing
-/// has been deleted, returns false.
-pub async fn delete_device_expired_messages(context: &Context) -> Result<bool, Error> {
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
-        let threshold_timestamp = time() - delete_device_after;
-
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .await
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .await
-            .unwrap_or_default()
-            .0;
-
-        // Delete expired messages
-        //
-        // Only update the rows that have to be updated, to avoid emitting
-        // unnecessary "chat modified" events.
-        let rows_modified = context
-            .sql
-            .execute(
-                "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
-             WHERE timestamp < ? \
-             AND chat_id > ? \
-             AND chat_id != ? \
-             AND chat_id != ?",
-                paramsv![
-                    DC_CHAT_ID_TRASH,
-                    threshold_timestamp,
-                    DC_CHAT_ID_LAST_SPECIAL,
-                    self_chat_id,
-                    device_chat_id
-                ],
-            )
-            .await?;
-
-        Ok(rows_modified > 0)
-    } else {
-        Ok(false)
-    }
-}
-
 pub async fn get_chat_media(
     context: &Context,
     chat_id: ChatId,
@@ -1930,20 +1983,19 @@ pub async fn create_group_chat(
         .sql
         .get_rowid(context, "chats", "grpid", grpid)
         .await?;
-    let chat_id = ChatId::new(row_id);
-    if !chat_id.is_error() {
-        if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
-            let mut draft_msg = Message::new(Viewtype::Text);
-            draft_msg.set_text(Some(draft_txt));
-            chat_id.set_draft_raw(context, &mut draft_msg).await;
-        }
 
-        context.emit_event(Event::MsgsChanged {
-            msg_id: MsgId::new(0),
-            chat_id: ChatId::new(0),
-        });
+    let chat_id = ChatId::new(row_id);
+    if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
+        let mut draft_msg = Message::new(Viewtype::Text);
+        draft_msg.set_text(Some(draft_txt));
+        chat_id.set_draft_raw(context, &mut draft_msg).await;
     }
 
+    context.emit_event(Event::MsgsChanged {
+        msg_id: MsgId::new(0),
+        chat_id: ChatId::new(0),
+    });
+
     Ok(chat_id)
 }
 
@@ -2580,7 +2632,7 @@ pub async fn forward_msgs(
                         .set(Param::PrepForwards, new_msg_id.to_u32().to_string());
                 }
 
-                msg.save_param_to_disk(context).await;
+                msg.update_param(context).await;
                 msg.param = save_param;
             } else {
                 msg.state = MessageState::OutPending;
@@ -2656,10 +2708,12 @@ pub(crate) async fn get_chat_id_by_grpid(
 /// Adds a message to device chat.
 ///
 /// Optional `label` can be provided to ensure that message is added only once.
-pub async fn add_device_msg(
+/// If `important` is true, a notification will be sent.
+pub async fn add_device_msg_with_importance(
     context: &Context,
     label: Option<&str>,
     msg: Option<&mut Message>,
+    important: bool,
 ) -> Result<MsgId, Error> {
     ensure!(
         label.is_some() || msg.is_some(),
@@ -2719,12 +2773,24 @@ pub async fn add_device_msg(
     }
 
     if !msg_id.is_unset() {
-        context.emit_event(Event::IncomingMsg { chat_id, msg_id });
+        if important {
+            context.emit_event(Event::IncomingMsg { chat_id, msg_id });
+        } else {
+            context.emit_event(Event::MsgsChanged { chat_id, msg_id });
+        }
     }
 
     Ok(msg_id)
 }
 
+pub async fn add_device_msg(
+    context: &Context,
+    label: Option<&str>,
+    msg: Option<&mut Message>,
+) -> Result<MsgId, Error> {
+    add_device_msg_with_importance(context, label, msg, false).await
+}
+
 pub async fn was_device_msg_ever_added(context: &Context, label: &str) -> Result<bool, Error> {
     ensure!(!label.is_empty(), "empty label");
     if let Ok(()) = context
@@ -2767,9 +2833,16 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
 /// For example, it can be a message showing that a member was added to a group.
 pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
+    let ephemeral_timer = match chat_id.get_ephemeral_timer(context).await {
+        Err(e) => {
+            warn!(context, "Could not get timer for info msg: {}", e);
+            return;
+        }
+        Ok(ephemeral_timer) => ephemeral_timer,
+    };
 
-    if context.sql.execute(
-        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid) VALUES (?,?,?, ?,?,?, ?,?);",
+    if let Err(e) = context.sql.execute(
+        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid,ephemeral_timer) VALUES (?,?,?, ?,?,?, ?,?,?);",
         paramsv![
             chat_id,
             DC_CONTACT_ID_INFO,
@@ -2779,8 +2852,10 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
             MessageState::InNoticed,
             text.as_ref().to_string(),
             rfc724_mid,
+            ephemeral_timer
         ]
-    ).await.is_err() {
+    ).await {
+        warn!(context, "Could not add info msg: {}", e);
         return;
     }
 
@@ -2804,7 +2879,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_chat_info() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let bob = Contact::create(&t.ctx, "bob", "bob@example.com")
             .await
             .unwrap();
@@ -2827,7 +2902,8 @@ mod tests {
                 "color": 15895624,
                 "profile_image": "",
                 "draft": "",
-                "is_muted": false
+                "is_muted": false,
+                "ephemeral_timer": "Disabled"
             }
         "#;
 
@@ -2838,7 +2914,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_draft_no_draft() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2848,7 +2924,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_draft_special_chat_id() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let draft = ChatId::new(DC_CHAT_ID_LAST_SPECIAL)
             .get_draft(&t.ctx)
             .await
@@ -2860,14 +2936,14 @@ mod tests {
     async fn test_get_draft_no_chat() {
         // This is a weird case, maybe this should be an error but we
         // do not get this info from the database currently.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let draft = ChatId::new(42).get_draft(&t.ctx).await.unwrap();
         assert!(draft.is_none());
     }
 
     #[async_std::test]
     async fn test_get_draft() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2883,7 +2959,7 @@ mod tests {
     #[async_std::test]
     async fn test_add_contact_to_chat_ex_add_self() {
         // Adding self to a contact should succeed, even though it's pointless.
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -2895,7 +2971,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_self_talk() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2916,7 +2992,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_deaddrop_chat() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat = Chat::load_from_db(&t.ctx, ChatId::new(DC_CHAT_ID_DEADDROP))
             .await
             .unwrap();
@@ -2931,7 +3007,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_unlabelled() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         // add two device-messages
         let mut msg1 = Message::new(Viewtype::Text);
@@ -2966,7 +3042,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_labelled() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         // add two device-messages with the same label (second attempt is not added)
         let mut msg1 = Message::new(Viewtype::Text);
@@ -3020,7 +3096,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_label_only() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let res = add_device_msg(&t.ctx, Some(""), None).await;
         assert!(res.is_err());
         let res = add_device_msg(&t.ctx, Some("some-label"), None).await;
@@ -3040,7 +3116,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_was_device_msg_ever_added() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         add_device_msg(&t.ctx, Some("some-label"), None).await.ok();
         assert!(was_device_msg_ever_added(&t.ctx, "some-label")
             .await
@@ -3064,7 +3140,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_delete_device_chat() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("message text".to_string());
@@ -3084,7 +3160,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_device_chat_cannot_sent() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
         let (device_chat_id, _) =
             create_or_lookup_by_contact_id(&t.ctx, DC_CONTACT_ID_DEVICE, Blocked::Not)
@@ -3104,7 +3180,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_delete_and_reset_all_device_msgs() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("message text".to_string());
         let msg_id1 = add_device_msg(&t.ctx, Some("some-label"), Some(&mut msg))
@@ -3141,7 +3217,7 @@ mod tests {
     #[async_std::test]
     async fn test_archive() {
         // create two chats
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("foo".to_string());
         let msg_id = add_device_msg(&t.ctx, None, Some(&mut msg)).await.unwrap();
@@ -3255,7 +3331,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_pinned() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // create 3 chats, wait 1 second in between to get a reliable order (we order by time)
         let mut msg = Message::new(Viewtype::Text);
@@ -3314,7 +3390,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_chat_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3338,7 +3414,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_same_chat_twice() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let contact1 = Contact::create(&context.ctx, "bob", "bob@mail.de")
             .await
             .unwrap();
@@ -3357,7 +3433,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_shall_attach_selfavatar() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3381,7 +3457,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_mute_duration() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3449,7 +3525,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_parent_is_encrypted() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();

src/chatlist.rs

@@ -5,6 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
+use crate::ephemeral::delete_expired_messages;
 use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
@@ -76,7 +77,7 @@ impl Chatlist {
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
     ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+    ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -99,7 +100,7 @@ impl Chatlist {
 
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context).await {
+        if let Err(err) = delete_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -147,11 +148,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
@@ -173,11 +175,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
@@ -206,11 +209,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND c.name LIKE ?3
@@ -236,11 +240,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND NOT c.archived=?3
@@ -424,7 +429,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_try_load() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
@@ -472,7 +477,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
         create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
@@ -497,7 +502,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_search_special_chat_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
 
         let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
@@ -530,7 +535,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_summary_unwrap() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();

src/config.rs

@@ -104,6 +104,9 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
+    ConfiguredInboxFolder,
+    ConfiguredMvboxFolder,
+    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -114,9 +117,21 @@ pub enum Config {
 
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
+
+    #[strum(props(default = "0"))]
+    /// Whether we send a warning if the password is wrong (set to false when we send a warning
+    /// because we do not want to send a second warning)
+    NotifyAboutWrongPw,
+
+    /// address to webrtc instance to use for videochats
+    WebrtcInstance,
 }
 
 impl Context {
+    pub async fn config_exists(&self, key: Config) -> bool {
+        self.sql.get_raw_config(self, key).await.is_some()
+    }
+
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
     pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
@@ -137,6 +152,7 @@ impl Context {
         // Default values
         match key {
             Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
@@ -197,21 +213,6 @@ impl Context {
                     None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
-            Config::InboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_inbox(false).await;
-                ret
-            }
-            Config::SentboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_sentbox(false).await;
-                ret
-            }
-            Config::MvboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_mvbox(false).await;
-                ret
-            }
             Config::Selfstatus => {
                 let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
@@ -281,7 +282,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_outside_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         File::create(&avatar_src)
@@ -310,7 +311,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_in_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.ctx.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
@@ -336,7 +337,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
         File::create(&avatar_src)
@@ -360,7 +361,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_media_quality_config_option() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure/mod.rs

@@ -68,16 +68,21 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let was_configured_before = self.is_configured().await;
         let mut param = LoginParam::from_database(self, "").await;
         let success = configure(self, &mut param).await;
+        self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
         if let Some(provider) = provider::get_provider_info(&param.addr) {
-            if !was_configured_before {
-                if let Some(config_defaults) = &provider.config_defaults {
-                    for def in config_defaults.iter() {
+            if let Some(config_defaults) = &provider.config_defaults {
+                for def in config_defaults.iter() {
+                    if !self.config_exists(def.key).await {
                         info!(self, "apply config_defaults {}={}", def.key, def.value);
                         self.set_config(def.key, Some(def.value)).await?;
+                    } else {
+                        info!(
+                            self,
+                            "skip already set config_defaults {}={}", def.key, def.value
+                        );
                     }
                 }
             }
@@ -96,11 +101,12 @@ impl Context {
 
         match success {
             Ok(_) => {
+                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+                    .await?;
                 progress!(self, 1000);
                 Ok(())
             }
             Err(err) => {
-                error!(self, "Configure Failed: {}", err);
                 progress!(self, 0);
                 Err(err)
             }
@@ -272,9 +278,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
         || ctx.get_config_bool(Config::MvboxMove).await;
 
-    imap.configure_folders(ctx, create_mvbox)
-        .await
-        .context("configuring folders failed")?;
+    imap.configure_folders(ctx, create_mvbox).await?;
 
     imap.select_with_uidvalidity(ctx, "INBOX")
         .await
@@ -455,7 +459,10 @@ async fn try_imap_connections(
         .await
         .is_ok()
     {
-        return Ok(()); // we directly return here if it was autoconfig or the connection succeeded
+        return Ok(());
+    }
+    if was_autoconfig {
+        bail!("autoconfig did not succeed");
     }
 
     progress!(context, 670);
@@ -489,7 +496,7 @@ async fn try_imap_connection(
         return Ok(());
     }
     if was_autoconfig {
-        return Ok(());
+        bail!("autoconfig did not succeed");
     }
 
     progress!(context, 650 + variation * 30);
@@ -549,7 +556,7 @@ async fn try_smtp_connections(
         return Ok(());
     }
     if was_autoconfig {
-        return Ok(());
+        bail!("No SMTP connection");
     }
     progress!(context, 850);
 
@@ -623,7 +630,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_no_panic_on_bad_credentials() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::Addr, Some("probably@unexistant.addr"))
             .await
@@ -637,7 +644,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_offline_autoconfig() {
-        let context = dummy_context().await.ctx;
+        let context = TestContext::new().await.ctx;
 
         let mut params = LoginParam::new();
         params.addr = "someone123@example.org".to_string();

src/constants.rs

@@ -84,6 +84,19 @@ impl Default for KeyGenType {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(i8)]
+pub enum VideochatType {
+    Unknown = 0,
+    BasicWebrtc = 1,
+}
+
+impl Default for VideochatType {
+    fn default() -> Self {
+        VideochatType::Unknown
+    }
+}
+
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
@@ -227,6 +240,10 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
+// max. width/height of images
+pub const BALANCED_IMAGE_SIZE: u32 = 1280;
+pub const WORSE_IMAGE_SIZE: u32 = 640;
+
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
@@ -292,6 +309,9 @@ pub enum Viewtype {
     /// The file is set via dc_msg_set_file()
     /// and retrieved via dc_msg_get_file().
     File = 60,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation = 70,
 }
 
 impl Default for Viewtype {
@@ -319,54 +339,6 @@ const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
 const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
 const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
 const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
-
-// TODO: Strings need some doumentation about used placeholders.
-// These constants are used to set stock translation strings
-
-const DC_STR_NOMESSAGES: usize = 1;
-const DC_STR_SELF: usize = 2;
-const DC_STR_DRAFT: usize = 3;
-const DC_STR_VOICEMESSAGE: usize = 7;
-const DC_STR_DEADDROP: usize = 8;
-const DC_STR_IMAGE: usize = 9;
-const DC_STR_VIDEO: usize = 10;
-const DC_STR_AUDIO: usize = 11;
-const DC_STR_FILE: usize = 12;
-const DC_STR_STATUSLINE: usize = 13;
-const DC_STR_NEWGROUPDRAFT: usize = 14;
-const DC_STR_MSGGRPNAME: usize = 15;
-const DC_STR_MSGGRPIMGCHANGED: usize = 16;
-const DC_STR_MSGADDMEMBER: usize = 17;
-const DC_STR_MSGDELMEMBER: usize = 18;
-const DC_STR_MSGGROUPLEFT: usize = 19;
-const DC_STR_GIF: usize = 23;
-const DC_STR_ENCRYPTEDMSG: usize = 24;
-const DC_STR_E2E_AVAILABLE: usize = 25;
-const DC_STR_ENCR_TRANSP: usize = 27;
-const DC_STR_ENCR_NONE: usize = 28;
-const DC_STR_CANTDECRYPT_MSG_BODY: usize = 29;
-const DC_STR_FINGERPRINTS: usize = 30;
-const DC_STR_READRCPT: usize = 31;
-const DC_STR_READRCPT_MAILBODY: usize = 32;
-const DC_STR_MSGGRPIMGDELETED: usize = 33;
-const DC_STR_E2E_PREFERRED: usize = 34;
-const DC_STR_CONTACT_VERIFIED: usize = 35;
-const DC_STR_CONTACT_NOT_VERIFIED: usize = 36;
-const DC_STR_CONTACT_SETUP_CHANGED: usize = 37;
-const DC_STR_ARCHIVEDCHATS: usize = 40;
-const DC_STR_STARREDMSGS: usize = 41;
-const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
-const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_CANNOT_LOGIN: usize = 60;
-const DC_STR_SERVER_RESPONSE: usize = 61;
-const DC_STR_MSGACTIONBYUSER: usize = 62;
-const DC_STR_MSGACTIONBYME: usize = 63;
-const DC_STR_MSGLOCATIONENABLED: usize = 64;
-const DC_STR_MSGLOCATIONDISABLED: usize = 65;
-const DC_STR_LOCATION: usize = 66;
-const DC_STR_STICKER: usize = 67;
-const DC_STR_COUNT: usize = 67;
 
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 

src/contact.rs

@@ -1029,10 +1029,10 @@ pub fn addr_normalize(addr: &str) -> &str {
     let norm = addr.trim();
 
     if norm.starts_with("mailto:") {
-        return &norm[7..];
+        norm.get(7..).unwrap_or(norm)
+    } else {
+        norm
     }
-
-    norm
 }
 
 fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
@@ -1042,11 +1042,15 @@ fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (Stri
     if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.as_ref().is_empty() {
-                normalize_name(&captures[1])
+                captures
+                    .get(1)
+                    .map_or("".to_string(), |m| normalize_name(m.as_str()))
             } else {
                 name.as_ref().to_string()
             },
-            captures[2].to_string(),
+            captures
+                .get(2)
+                .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
         (name.as_ref().to_string(), addr.as_ref().to_string())
@@ -1085,21 +1089,45 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
     }
 }
 
+/// Set profile image for a contact.
+///
+/// The given profile image is expected to be already in the blob directory
+/// as profile images can be set only by receiving messages, this should be always the case, however.
+///
+/// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
+/// this typically happens if we see message with our own profile image, sent from another device.
 pub(crate) async fn set_profile_image(
     context: &Context,
     contact_id: u32,
     profile_image: &AvatarAction,
+    was_encrypted: bool,
 ) -> Result<()> {
-    // the given profile image is expected to be already in the blob directory
-    // as profile images can be set only by receiving messages, this should be always the case, however.
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            contact.param.set(Param::ProfileImage, profile_image);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context
+                        .set_config(Config::Selfavatar, Some(profile_image))
+                        .await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar.");
+                }
+            } else {
+                contact.param.set(Param::ProfileImage, profile_image);
+            }
             true
         }
         AvatarAction::Delete => {
-            contact.param.remove(Param::ProfileImage);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context.set_config(Config::Selfavatar, None).await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar deletion.");
+                }
+            } else {
+                contact.param.remove(Param::ProfileImage);
+            }
             true
         }
     };
@@ -1113,38 +1141,21 @@ pub(crate) async fn set_profile_image(
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
-/// - Convert names as "Petersen, Björn" to "Björn Petersen"
 /// - Trims the resulting string
 ///
 /// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
 pub fn normalize_name(full_name: impl AsRef<str>) -> String {
-    let mut full_name = full_name.as_ref().trim();
+    let full_name = full_name.as_ref().trim();
     if full_name.is_empty() {
         return full_name.into();
     }
 
-    let len = full_name.len();
-    if len > 1 {
-        let firstchar = full_name.as_bytes()[0];
-        let lastchar = full_name.as_bytes()[len - 1];
-        if firstchar == b'\'' && lastchar == b'\''
-            || firstchar == b'\"' && lastchar == b'\"'
-            || firstchar == b'<' && lastchar == b'>'
-        {
-            full_name = &full_name[1..len - 1];
-        }
+    match full_name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => full_name
+            .get(1..full_name.len() - 1)
+            .map_or("".to_string(), |s| s.trim().into()),
+        _ => full_name.to_string(),
     }
-
-    if let Some(p1) = full_name.find(',') {
-        let (last_name, first_name) = full_name.split_at(p1);
-
-        let last_name = last_name.trim();
-        let first_name = (&first_name[1..]).trim();
-
-        return format!("{} {}", first_name, last_name);
-    }
-
-    full_name.trim().into()
 }
 
 fn cat_fingerprint(
@@ -1235,7 +1246,6 @@ mod tests {
 
     #[test]
     fn test_normalize_name() {
-        assert_eq!(&normalize_name("Doe, John"), "John Doe");
         assert_eq!(&normalize_name(" hello world   "), "hello world");
         assert_eq!(&normalize_name("<"), "<");
         assert_eq!(&normalize_name(">"), ">");
@@ -1270,7 +1280,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_contacts() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let contacts = Contact::get_all(&context.ctx, 0, Some("some2"))
             .await
             .unwrap();
@@ -1294,10 +1304,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_self_addr() -> Result<()> {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         assert!(t.ctx.is_self_addr("me@me.org").await.is_err());
 
-        let addr = configure_alice_keypair(&t.ctx).await;
+        let addr = t.configure_alice().await;
         assert_eq!(t.ctx.is_self_addr("me@me.org").await?, false);
         assert_eq!(t.ctx.is_self_addr(&addr).await?, true);
 
@@ -1307,7 +1317,7 @@ mod tests {
     #[async_std::test]
     async fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let book = concat!(
             "  Name one  \n one@eins.org \n",
             "Name two\ntwo@deux.net\n",
@@ -1400,10 +1410,10 @@ mod tests {
         assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
-        assert_eq!(contact.get_name(), "Alice Wonderland");
-        assert_eq!(contact.get_display_name(), "Alice Wonderland");
+        assert_eq!(contact.get_name(), "Wonderland, Alice");
+        assert_eq!(contact.get_display_name(), "Wonderland, Alice");
         assert_eq!(contact.get_addr(), "alice@w.de");
-        assert_eq!(contact.get_name_n_addr(), "Alice Wonderland (alice@w.de)");
+        assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
 
         // check SELF
         let contact = Contact::load_from_db(&t.ctx, DC_CONTACT_ID_SELF)
@@ -1420,7 +1430,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
@@ -1483,7 +1493,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_create_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t.ctx, "", "claire@example.org")
@@ -1530,7 +1540,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_edit_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "dave@example.org"
         let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org")
@@ -1574,7 +1584,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_name_in_address() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let contact_id = Contact::create(&t.ctx, "", "<dave@example.org>")
             .await
@@ -1587,7 +1597,7 @@ mod tests {
             .await
             .unwrap();
         let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
-        assert_eq!(contact.get_name(), "Dave Mueller");
+        assert_eq!(contact.get_name(), "Mueller, Dave");
         assert_eq!(contact.get_addr(), "dave@example.org");
 
         let contact_id = Contact::create(&t.ctx, "name1", "name2 <dave@example.org>")

src/context.rs

@@ -6,6 +6,7 @@ use std::ops::Deref;
 
 use async_std::path::{Path, PathBuf};
 use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use async_std::task;
 
 use crate::chat::*;
 use crate::config::Config;
@@ -14,12 +15,10 @@ use crate::contact::*;
 use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::{Event, EventEmitter, Events};
-use crate::job::{self, Action};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message, MessengerMessage, MsgId};
-use crate::param::Params;
+use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
 use crate::sql::Sql;
 use std::time::SystemTime;
@@ -52,10 +51,13 @@ pub struct InnerContext {
     pub(crate) generating_key_mutex: Mutex<()>,
     /// Mutex to enforce only a single running oauth2 is running.
     pub(crate) oauth2_mutex: Mutex<()>,
+    /// Mutex to prevent a race condition when a "your pw is wrong" warning is sent, resulting in multiple messeges being sent.
+    pub(crate) wrong_pw_warning_mutex: Mutex<()>,
     pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
     creation_time: SystemTime,
 }
@@ -118,9 +120,11 @@ impl Context {
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
+            wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
         };
 
@@ -300,13 +304,11 @@ impl Context {
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
+            .get_config(Config::ConfiguredSentboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
+            .get_config(Config::ConfiguredMvboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
 
@@ -442,61 +444,19 @@ impl Context {
             .unwrap_or_default()
     }
 
-    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        folder_name.as_ref() == "INBOX"
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
-            .await;
-        if let Some(name) = sentbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
-            .await;
-
-        if let Some(name) = mvbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
-    }
-
-    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove).await {
-            return;
-        }
-
-        if self.is_mvbox(folder).await {
-            return;
-        }
-        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
-            if msg.is_setupmessage() {
-                // do not move setup messages;
-                // there may be a non-delta device that wants to handle it
-                return;
-            }
-
-            match msg.is_dc_message {
-                MessengerMessage::No => {}
-                MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job::add(
-                        self,
-                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
-                    )
-                    .await;
-                }
-            }
-        }
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 }
 
@@ -556,7 +516,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_fresh_msgs() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let fresh = t.ctx.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
@@ -611,13 +571,13 @@ mod tests {
 
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         std::mem::drop(t.ctx);
     }
 
     #[async_std::test]
     async fn test_get_info() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let info = t.ctx.get_info().await;
         assert!(info.get("database_dir").is_some());

src/dc_tools.rs

@@ -4,6 +4,7 @@
 use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
+use std::io::Cursor;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
@@ -22,6 +23,7 @@ pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let ellipse = "[...]";
 
@@ -54,6 +56,7 @@ const COLORS: [u32; 16] = [
     0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
 ];
 
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let str_lower = s.as_ref().to_lowercase();
     let mut checksum = 0;
@@ -198,7 +201,7 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
 pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str) -> String {
     let hostname = from_addr
         .find('@')
-        .map(|k| &from_addr[k..])
+        .and_then(|k| from_addr.get(k..))
         .unwrap_or("@nohost");
     match grpid {
         Some(grpid) => format!("Gr.{}.{}{}", grpid, dc_create_id(), hostname),
@@ -240,9 +243,9 @@ pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
 
 /// Returns the `(width, height)` of the given image buffer.
 pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
-    let meta = image_meta::load_from_buf(buf)?;
-
-    Ok((meta.dimensions.width, meta.dimensions.height))
+    let image = image::io::Reader::new(Cursor::new(buf));
+    let dimensions = image.into_dimensions()?;
+    Ok(dimensions)
 }
 
 /// Expand paths relative to $BLOBDIR into absolute paths.
@@ -774,7 +777,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_file_handling() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let context = &t.ctx;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
@@ -853,7 +856,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_ne!(
             dc_create_smeared_timestamp(&t.ctx).await,
             dc_create_smeared_timestamp(&t.ctx).await
@@ -869,7 +872,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamps() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
         let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
         let next = dc_smeared_time(&t.ctx).await;

src/e2ee.rs

@@ -11,7 +11,7 @@ use crate::context::Context;
 use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, SignedPublicKey, SignedSecretKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -115,11 +115,17 @@ impl EncryptHelper {
     }
 }
 
+/// Tries to decrypt a message, but only if it is structured as an
+/// Autocrypt message, i.e.  encrypted and signed with a valid
+/// signature.
+///
+/// Returns decrypted body and a set of valid signature fingerprints
+/// if successful.
 pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
@@ -187,24 +193,23 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
         "Not a multipart/encrypted message: {}",
         mail.ctype.mimetype
     );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
+    if let [first_part, second_part] = &mail.subparts[..] {
+        ensure!(
+            first_part.ctype.mimetype == "application/pgp-encrypted",
+            "Invalid Autocrypt Level 1 version part: {:?}",
+            first_part.ctype,
+        );
 
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
+        ensure!(
+            second_part.ctype.mimetype == "application/octet-stream",
+            "Invalid Autocrypt Level 1 encrypted part: {:?}",
+            second_part.ctype
+        );
 
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
-    );
-
-    Ok(&mail.subparts[1])
+        Ok(second_part)
+    } else {
+        bail!("Invalid Autocrypt Level 1 Mime Parts")
+    }
 }
 
 async fn decrypt_if_autocrypt_message<'a>(
@@ -212,7 +217,7 @@ async fn decrypt_if_autocrypt_message<'a>(
     mail: &ParsedMail<'a>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<String>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     //  The returned bool is true if we detected an Autocrypt-encrypted
     // message and successfully decrypted it. Decryption then modifies the
@@ -244,7 +249,7 @@ async fn decrypt_part(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<String>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
@@ -267,6 +272,7 @@ async fn decrypt_part(
     Ok(None)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     if let Some(index) = input.iter().position(|b| *b > b' ') {
         if input.len() - index > 26 {
@@ -327,14 +333,14 @@ mod tests {
 
         #[async_std::test]
         async fn test_prexisting() {
-            let t = dummy_context().await;
-            let test_addr = configure_alice_keypair(&t.ctx).await;
+            let t = TestContext::new().await;
+            let test_addr = t.configure_alice().await;
             assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
-            let t = dummy_context().await;
+            let t = TestContext::new().await;
             assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
         }
     }

src/ephemeral.rs

@@ -0,0 +1,528 @@
+//! # Ephemeral messages
+//!
+//! Ephemeral messages are messages that have an Ephemeral-Timer
+//! header attached to them, which specifies time in seconds after
+//! which the message should be deleted both from the device and from
+//! the server. The timer is started when the message is marked as
+//! seen, which usually happens when its contents is displayed on
+//! device screen.
+//!
+//! Each chat, including 1:1, group chats and "saved messages" chat,
+//! has its own ephemeral timer setting, which is applied to all
+//! messages sent to the chat. The setting is synchronized to all the
+//! devices participating in the chat by applying the timer value from
+//! all received messages, including BCC-self ones, to the chat. This
+//! way the setting is eventually synchronized among all participants.
+//!
+//! When user changes ephemeral timer setting for the chat, a system
+//! message is automatically sent to update the setting for all
+//! participants. This allows changing the setting for a chat like any
+//! group chat setting, e.g. name and avatar, without the need to
+//! write an actual message.
+//!
+//! ## Device settings
+//!
+//! In addition to per-chat ephemeral message setting, each device has
+//! two global user-configured settings that complement per-chat
+//! settings: `delete_device_after` and `delete_server_after`. These
+//! settings are not synchronized among devices and apply to all
+//! messages known to the device, including messages sent or received
+//! before configuring the setting.
+//!
+//! `delete_device_after` configures the maximum time device is
+//! storing the messages locally. `delete_server_after` configures the
+//! time after which device will delete the messages it knows about
+//! from the server.
+//!
+//! ## How messages are deleted
+//!
+//! When the message is deleted locally, its contents is removed and
+//! it is moved to the trash chat. This database entry is then used to
+//! track the Message-ID and corresponding IMAP folder and UID until
+//! the message is deleted from the server. Vice versa, when device
+//! deletes the message from the server, it removes IMAP folder and
+//! UID information, but keeps the message contents. When database
+//! entry is both moved to trash chat and does not contain UID
+//! information, it is deleted from the database, leaving no trace of
+//! the message.
+//!
+//! ## When messages are deleted
+//!
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
+//!
+//! Server deletion happens by generating IMAP deletion jobs based on
+//! the database entries which are expired either according to their
+//! ephemeral message timers or global `delete_server_after` setting.
+
+use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use crate::constants::{
+    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::error::{ensure, Error};
+use crate::events::Event;
+use crate::message::{Message, MessageState, MsgId};
+use crate::mimeparser::SystemMessage;
+use crate::sql;
+use crate::stock::StockMessage;
+use async_std::task;
+use serde::{Deserialize, Serialize};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
+pub enum Timer {
+    Disabled,
+    Enabled { duration: u32 },
+}
+
+impl Timer {
+    pub fn to_u32(self) -> u32 {
+        match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => duration,
+        }
+    }
+
+    pub fn from_u32(duration: u32) -> Self {
+        if duration == 0 {
+            Self::Disabled
+        } else {
+            Self::Enabled { duration }
+        }
+    }
+}
+
+impl Default for Timer {
+    fn default() -> Self {
+        Self::Disabled
+    }
+}
+
+impl ToString for Timer {
+    fn to_string(&self) -> String {
+        self.to_u32().to_string()
+    }
+}
+
+impl FromStr for Timer {
+    type Err = ParseIntError;
+
+    fn from_str(input: &str) -> Result<Timer, ParseIntError> {
+        input.parse::<u32>().map(Self::from_u32)
+    }
+}
+
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
+    }
+}
+
+impl ChatId {
+    /// Get ephemeral message timer value in seconds.
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+        let timer = context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
+            )
+            .await?;
+        Ok(timer.unwrap_or_default())
+    }
+
+    /// Set ephemeral timer value without sending a message.
+    ///
+    /// Used when a message arrives indicating that someone else has
+    /// changed the timer value for a chat.
+    pub(crate) async fn inner_set_ephemeral_timer(
+        self,
+        context: &Context,
+        timer: Timer,
+    ) -> Result<(), Error> {
+        ensure!(!self.is_special(), "Invalid chat ID");
+
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+             SET ephemeral_timer=?
+             WHERE id=?;",
+                paramsv![timer, self],
+            )
+            .await?;
+
+        context.emit_event(Event::ChatEphemeralTimerModified {
+            chat_id: self,
+            timer,
+        });
+        Ok(())
+    }
+
+    /// Set ephemeral message timer value in seconds.
+    ///
+    /// If timer value is 0, disable ephemeral message timer.
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+        if timer == self.get_ephemeral_timer(context).await? {
+            return Ok(());
+        }
+        self.inner_set_ephemeral_timer(context, timer).await?;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
+        if let Err(err) = send_msg(context, self, &mut msg).await {
+            error!(
+                context,
+                "Failed to send a message about ephemeral message timer change: {:?}", err
+            );
+        }
+        Ok(())
+    }
+}
+
+/// Returns a stock message saying that ephemeral timer is changed to `timer` by `from_id`.
+pub(crate) async fn stock_ephemeral_timer_changed(
+    context: &Context,
+    timer: Timer,
+    from_id: u32,
+) -> String {
+    let stock_message = match timer {
+        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+        Timer::Enabled { duration } => match duration {
+            60 => StockMessage::MsgEphemeralTimerMinute,
+            3600 => StockMessage::MsgEphemeralTimerHour,
+            86400 => StockMessage::MsgEphemeralTimerDay,
+            604_800 => StockMessage::MsgEphemeralTimerWeek,
+            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
+            _ => StockMessage::MsgEphemeralTimerEnabled,
+        },
+    };
+
+    context
+        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
+        .await
+}
+
+impl MsgId {
+    /// Returns ephemeral message timer value for the message.
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
+        let res = match context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
+            )
+            .await?
+        {
+            None | Some(0) => Timer::Disabled,
+            Some(duration) => Timer::Enabled { duration },
+        };
+        Ok(res)
+    }
+
+    /// Starts ephemeral message timer for the message if it is not started yet.
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
+        if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
+            let ephemeral_timestamp = time() + i64::from(duration);
+
+            context
+                .sql
+                .execute(
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
+                WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
+                AND id = ?",
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
+                )
+                .await?;
+            schedule_ephemeral_task(context).await;
+        }
+        Ok(())
+    }
+}
+
+/// Deletes messages which are expired according to
+/// `delete_device_after` setting or `ephemeral_timestamp` column.
+///
+/// Returns true if any message is deleted, so caller can emit
+/// MsgsChanged event. If nothing has been deleted, returns
+/// false. This function does not emit the MsgsChanged event itself,
+/// because it is also called when chatlist is reloaded, and emitting
+/// MsgsChanged there will cause infinite reload loop.
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+    let mut updated = context
+        .sql
+        .execute(
+            "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE \
+             ephemeral_timestamp != 0 \
+             AND ephemeral_timestamp < ? \
+             AND chat_id != ?",
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+        )
+        .await?
+        > 0;
+
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
+        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap_or_default()
+            .0;
+        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+            .await
+            .unwrap_or_default()
+            .0;
+
+        let threshold_timestamp = time() - delete_device_after;
+
+        // Delete expired messages
+        //
+        // Only update the rows that have to be updated, to avoid emitting
+        // unnecessary "chat modified" events.
+        let rows_modified = context
+            .sql
+            .execute(
+                "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE timestamp < ? \
+             AND chat_id > ? \
+             AND chat_id != ? \
+             AND chat_id != ?",
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
+            )
+            .await?;
+
+        updated |= rows_modified > 0;
+    }
+
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
+}
+
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
+///
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
+    let ephemeral_timestamp: Option<i64> = match context
+        .sql
+        .query_get_value_result(
+            "SELECT ephemeral_timestamp \
+         FROM msgs \
+         WHERE ephemeral_timestamp != 0 \
+           AND chat_id != ? \
+         ORDER BY ephemeral_timestamp ASC \
+         LIMIT 1",
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
+        )
+        .await
+    {
+        Err(err) => {
+            warn!(context, "Can't calculate next ephemeral timeout: {}", err);
+            return;
+        }
+        Ok(ephemeral_timestamp) => ephemeral_timestamp,
+    };
+
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
+
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+        let now = SystemTime::now();
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
+
+        if let Ok(duration) = until.duration_since(now) {
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                emit_event!(
+                    context1,
+                    Event::MsgsChanged {
+                        chat_id: ChatId::new(0),
+                        msg_id: MsgId::new(0)
+                    }
+                );
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            emit_event!(
+                context,
+                Event::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0)
+                }
+            );
+        }
+    }
+}
+
+/// Returns ID of any expired message that should be deleted from the server.
+///
+/// It looks up the trash chat too, to find messages that are already
+/// deleted locally, but not deleted on the server.
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+    let now = time();
+
+    let threshold_timestamp = match context.get_config_delete_server_after().await {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
+
+    context
+        .sql
+        .query_row_optional(
+            "SELECT id FROM msgs \
+         WHERE ( \
+         timestamp < ? \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         ) \
+         AND server_uid != 0 \
+         LIMIT 1",
+            paramsv![threshold_timestamp, now],
+            |row| row.get::<_, MsgId>(0),
+        )
+        .await
+}
+
+/// Start ephemeral timers for seen messages if they are not started
+/// yet.
+///
+/// It is possible that timers are not started due to a missing or
+/// failed `MsgId.start_ephemeral_timer()` call, either in the current
+/// or previous version of Delta Chat.
+///
+/// This function is supposed to be called in the background,
+/// e.g. from housekeeping task.
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+    context
+        .sql
+        .execute(
+            "UPDATE msgs \
+    SET ephemeral_timestamp = ? + ephemeral_timer \
+    WHERE ephemeral_timer > 0 \
+    AND ephemeral_timestamp = 0 \
+    AND state NOT IN (?, ?, ?)",
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
+        )
+        .await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::*;
+
+    #[async_std::test]
+    async fn test_stock_ephemeral_messages() {
+        let context = TestContext::new().await.ctx;
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            "Message deletion timer is disabled by me."
+        );
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
+            "Message deletion timer is disabled."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
+            "Message deletion timer is set to 1 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
+            "Message deletion timer is set to 30 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
+            "Message deletion timer is set to 1 minute."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
+            "Message deletion timer is set to 1 hour."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 day."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 week."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 4 * 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 4 weeks."
+        );
+    }
+}

src/events.rs

@@ -5,6 +5,7 @@ use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
 
 #[derive(Debug)]
@@ -139,7 +140,6 @@ pub enum Event {
     /// Network errors should be reported to users in a non-disturbing way,
     /// however, as network errors may come in a sequence,
     /// it is not useful to raise each an every error to the user.
-    /// For this purpose, data1 is set to 1 if the error is probably worth reporting.
     ///
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
@@ -189,9 +189,19 @@ pub enum Event {
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// This event does not include ephemeral timer modification, which
+    /// is a separate event.
     #[strum(props(id = "2020"))]
     ChatModified(ChatId),
 
+    /// Chat ephemeral timer changed.
+    #[strum(props(id = "2021"))]
+    ChatEphemeralTimerModified {
+        chat_id: ChatId,
+        timer: EphemeralTimer,
+    },
+
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.

src/headerdef.rs

@@ -21,6 +21,7 @@ pub enum HeaderDef {
     References,
     InReplyTo,
     Precedence,
+    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -34,6 +35,7 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
+    ChatWebrtcRoom,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,
@@ -41,6 +43,7 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    EphemeralTimer,
     _TestHeader,
 }
 

src/imap/client.rs

@@ -7,7 +7,7 @@ use async_imap::{
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
+use crate::login_param::dc_build_tls;
 
 use super::session::SessionStream;
 
@@ -78,10 +78,10 @@ impl Client {
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        certificate_checks: CertificateChecks,
+        strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
+        let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
@@ -118,16 +118,12 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
+    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {
             let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(certificate_checks);
+            let tls = dc_build_tls(strict_tls);
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();

src/imap/idle.rs

@@ -1,10 +1,11 @@
 use super::Imap;
 
 use async_imap::extensions::idle::IdleResponse;
+use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::context::Context;
+use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::select_folder;
 use super::session::Session;
@@ -13,19 +14,19 @@ type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
+    #[error("IMAP IDLE protocol failed to init/complete: {0}")]
     IdleProtocolFailed(#[from] async_imap::error::Error),
 
-    #[error("IMAP IDLE protocol timed out")]
+    #[error("IMAP IDLE protocol timed out: {0}")]
     IdleTimeout(#[from] async_std::future::TimeoutError),
 
     #[error("IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[error("IMAP select folder error")]
+    #[error("IMAP select folder error: {0}")]
     SelectFolderError(#[from] select_folder::Error),
 
-    #[error("Setup handle error")]
+    #[error("Setup handle error: {0}")]
     SetupHandleError(#[from] super::Error),
 }
 
@@ -34,7 +35,11 @@ impl Imap {
         self.config.can_idle
     }
 
-    pub async fn idle(&mut self, context: &Context, watch_folder: Option<String>) -> Result<bool> {
+    pub async fn idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> Result<InterruptInfo> {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
@@ -44,11 +49,27 @@ impl Imap {
 
         self.select_folder(context, watch_folder.clone()).await?;
 
-        let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
-        let mut probe_network = false;
+        let mut info = Default::default();
+
+        if let Some(session) = self.session.take() {
+            // if we have unsolicited responses we directly return
+            let mut unsolicited_exists = false;
+            while let Ok(response) = session.unsolicited_responses.try_recv() {
+                match response {
+                    UnsolicitedResponse::Exists(_) => {
+                        warn!(context, "skip idle, got unsolicited EXISTS {:?}", response);
+                        unsolicited_exists = true;
+                    }
+                    _ => info!(context, "ignoring unsolicited response {:?}", response),
+                }
+            }
+
+            if unsolicited_exists {
+                self.session = Some(session);
+                return Ok(info);
+            }
 
-        if let Some(session) = session {
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
                 return Err(Error::IdleProtocolFailed(err));
@@ -58,81 +79,56 @@ impl Imap {
 
             enum Event {
                 IdleResponse(IdleResponse),
-                Interrupt(bool),
+                Interrupt(InterruptInfo),
             }
 
-            if self.skip_next_idle_wait {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait = false;
-                drop(idle_wait);
+            info!(context, "Idle entering wait-on-remote state");
+            let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
+                let probe_network = self.idle_interrupt.recv().await;
+
+                // cancel imap idle connection properly
                 drop(interrupt);
 
-                info!(context, "Idle wait was skipped");
-            } else {
-                info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
-                    self.idle_interrupt.recv().map(|probe_network| {
-                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
-                    }),
-                );
+                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+            });
 
-                match fut.await {
-                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
-                        info!(context, "Idle has NewData");
-                    }
-                    // TODO: idle_wait does not distinguish manual interrupts
-                    // from Timeouts if we would know it's a Timeout we could bail
-                    // directly and reconnect .
-                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                        info!(context, "Idle-wait timeout or interruption");
-                    }
-                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Ok(Event::Interrupt(probe)) => {
-                        probe_network = probe;
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Err(err) => {
-                        warn!(context, "Idle wait errored: {:?}", err);
-                    }
+            match fut.await {
+                Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
+                    info!(context, "Idle has NewData {:?}", x);
+                }
+                Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                    info!(context, "Idle-wait timeout or interruption");
+                }
+                Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                    info!(context, "Idle wait was interrupted");
+                }
+                Ok(Event::Interrupt(i)) => {
+                    info = i;
+                    info!(context, "Idle wait was interrupted");
+                }
+                Err(err) => {
+                    warn!(context, "Idle wait errored: {:?}", err);
                 }
             }
 
-            // if we can't properly terminate the idle
-            // protocol let's break the connection.
-            let res = handle
+            let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(|err| {
-                    self.trigger_reconnect();
-                    Error::IdleTimeout(err)
-                })?;
-
-            match res {
-                Ok(session) => {
-                    self.session = Some(Session { inner: session });
-                }
-                Err(err) => {
-                    // if we cannot terminate IDLE it probably
-                    // means that we waited long (with idle_wait)
-                    // but the network went away/changed
-                    self.trigger_reconnect();
-                    return Err(Error::IdleProtocolFailed(err));
-                }
-            }
+                .map_err(Error::IdleTimeout)??;
+            self.session = Some(Session { inner: session });
+        } else {
+            warn!(context, "Attempted to idle without a session");
         }
 
-        Ok(probe_network)
+        Ok(info)
     }
 
     pub(crate) async fn fake_idle(
         &mut self,
         context: &Context,
         watch_folder: Option<String>,
-    ) -> bool {
+    ) -> InterruptInfo {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
@@ -144,73 +140,66 @@ impl Imap {
             return self.idle_interrupt.recv().await.unwrap_or_default();
         }
 
-        let mut probe_network = false;
-        if self.skip_next_idle_wait {
-            // interrupt_idle has happened before we
-            // provided self.interrupt
-            self.skip_next_idle_wait = false;
-            info!(context, "fake-idle wait was skipped");
-        } else {
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        // check every minute if there are new messages
+        // TODO: grow sleep durations / make them more flexible
+        let mut interval = async_std::stream::interval(Duration::from_secs(60));
 
-            enum Event {
-                Tick,
-                Interrupt(bool),
-            }
-            // loop until we are interrupted or if we fetched something
-            probe_network =
-                loop {
-                    use futures::future::FutureExt;
-                    match interval
-                        .next()
-                        .map(|_| Event::Tick)
-                        .race(self.idle_interrupt.recv().map(|probe_network| {
-                            Event::Interrupt(probe_network.unwrap_or_default())
-                        }))
-                        .await
-                    {
-                        Event::Tick => {
-                            // try to connect with proper login params
-                            // (setup_handle_if_needed might not know about them if we
-                            // never successfully connected)
-                            if let Err(err) = self.connect_configured(context).await {
-                                warn!(context, "fake_idle: could not connect: {}", err);
-                                continue;
-                            }
-                            if self.config.can_idle {
-                                // we only fake-idled because network was gone during IDLE, probably
-                                break false;
-                            }
-                            info!(context, "fake_idle is connected");
-                            // we are connected, let's see if fetching messages results
-                            // in anything.  If so, we behave as if IDLE had data but
-                            // will have already fetched the messages so perform_*_fetch
-                            // will not find any new.
+        enum Event {
+            Tick,
+            Interrupt(InterruptInfo),
+        }
+        // loop until we are interrupted or if we fetched something
+        let info = loop {
+            use futures::future::FutureExt;
+            match interval
+                .next()
+                .map(|_| Event::Tick)
+                .race(
+                    self.idle_interrupt
+                        .recv()
+                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                )
+                .await
+            {
+                Event::Tick => {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context).await {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break InterruptInfo::new(false, None);
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
 
-                            if let Some(ref watch_folder) = watch_folder {
-                                match self.fetch_new_messages(context, watch_folder).await {
-                                    Ok(res) => {
-                                        info!(context, "fetch_new_messages returned {:?}", res);
-                                        if res {
-                                            break false;
-                                        }
-                                    }
-                                    Err(err) => {
-                                        error!(context, "could not fetch from folder: {}", err);
-                                        self.trigger_reconnect()
-                                    }
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break InterruptInfo::new(false, None);
                                 }
                             }
-                        }
-                        Event::Interrupt(probe_network) => {
-                            // Interrupt
-                            break probe_network;
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
                         }
                     }
-                };
-        }
+                }
+                Event::Interrupt(info) => {
+                    // Interrupt
+                    break info;
+                }
+            }
+        };
 
         info!(
             context,
@@ -222,6 +211,6 @@ impl Imap {
                 / 1000.,
         );
 
-        probe_network
+        info
     }
 }

src/imap/mod.rs

@@ -23,11 +23,12 @@ use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
-use crate::message::{self, update_server_uid};
+use crate::message::{self, update_server_uid, MessageState};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
-use crate::stock::StockMessage;
+use crate::provider::get_provider_info;
+use crate::{chat, scheduler::InterruptInfo, stock::StockMessage};
 
 mod client;
 mod idle;
@@ -35,6 +36,7 @@ pub mod select_folder;
 mod session;
 
 use client::Client;
+use message::Message;
 use session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
@@ -109,13 +111,13 @@ const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
-    idle_interrupt: Receiver<bool>,
+    idle_interrupt: Receiver<InterruptInfo>,
     config: ImapConfig,
     session: Option<Session>,
     connected: bool,
     interrupt: Option<stop_token::StopSource>,
-    skip_next_idle_wait: bool,
     should_reconnect: bool,
+    login_failed_once: bool,
 }
 
 #[derive(Debug)]
@@ -149,7 +151,7 @@ struct ImapConfig {
     pub imap_port: u16,
     pub imap_user: String,
     pub imap_pw: String,
-    pub certificate_checks: CertificateChecks,
+    pub strict_tls: bool,
     pub server_flags: usize,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
@@ -169,7 +171,7 @@ impl Default for ImapConfig {
             imap_port: 0,
             imap_user: "".into(),
             imap_pw: "".into(),
-            certificate_checks: Default::default(),
+            strict_tls: false,
             server_flags: 0,
             selected_folder: None,
             selected_mailbox: None,
@@ -181,15 +183,15 @@ impl Default for ImapConfig {
 }
 
 impl Imap {
-    pub fn new(idle_interrupt: Receiver<bool>) -> Self {
+    pub fn new(idle_interrupt: Receiver<InterruptInfo>) -> Self {
         Imap {
             idle_interrupt,
             config: Default::default(),
             session: Default::default(),
             connected: Default::default(),
             interrupt: Default::default(),
-            skip_next_idle_wait: Default::default(),
             should_reconnect: Default::default(),
+            login_failed_once: Default::default(),
         }
     }
 
@@ -228,7 +230,7 @@ impl Imap {
                 match Client::connect_insecure((imap_server, imap_port)).await {
                     Ok(client) => {
                         if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.certificate_checks).await
+                            client.secure(imap_server, config.strict_tls).await
                         } else {
                             Ok(client)
                         }
@@ -240,12 +242,8 @@ impl Imap {
                 let imap_server: &str = config.imap_server.as_ref();
                 let imap_port = config.imap_port;
 
-                Client::connect_secure(
-                    (imap_server, imap_port),
-                    imap_server,
-                    config.certificate_checks,
-                )
-                .await
+                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
+                    .await
             };
 
         let login_res = match connection_res {
@@ -295,19 +293,43 @@ impl Imap {
 
         match login_res {
             Ok(session) => {
+                // needs to be set here to ensure it is set on reconnects.
+                self.connected = true;
                 self.session = Some(session);
+                self.login_failed_once = false;
                 Ok(())
             }
+
             Err((err, _)) => {
                 let imap_user = self.config.imap_user.to_owned();
                 let message = context
                     .stock_string_repl_str(StockMessage::CannotLogin, &imap_user)
                     .await;
 
-                emit_event!(
-                    context,
-                    Event::ErrorNetwork(format!("{} ({})", message, err))
-                );
+                warn!(context, "{} ({})", message, err);
+                emit_event!(context, Event::ErrorNetwork(message.clone()));
+
+                let lock = context.wrong_pw_warning_mutex.lock().await;
+                if self.login_failed_once
+                    && context.get_config_bool(Config::NotifyAboutWrongPw).await
+                {
+                    if let Err(e) = context.set_config(Config::NotifyAboutWrongPw, None).await {
+                        warn!(context, "{}", e);
+                    }
+                    drop(lock);
+
+                    let mut msg = Message::new(Viewtype::Text);
+                    msg.text = Some(message);
+                    if let Err(e) =
+                        chat::add_device_msg_with_importance(context, None, Some(&mut msg), true)
+                            .await
+                    {
+                        warn!(context, "{}", e);
+                    }
+                } else {
+                    self.login_failed_once = true;
+                }
+
                 self.trigger_reconnect();
                 Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
             }
@@ -315,9 +337,15 @@ impl Imap {
     }
 
     async fn unsetup_handle(&mut self, context: &Context) {
+        // Close folder if messages should be expunged
+        if let Err(err) = self.close_folder(context).await {
+            warn!(context, "failed to close folder: {:?}", err);
+        }
+
+        // Logout from the server
         if let Some(mut session) = self.session.take() {
-            if let Err(err) = session.close().await {
-                warn!(context, "failed to close connection: {:?}", err);
+            if let Err(err) = session.logout().await {
+                warn!(context, "failed to logout: {:?}", err);
             }
         }
         self.connected = false;
@@ -377,7 +405,15 @@ impl Imap {
             config.imap_port = imap_port;
             config.imap_user = imap_user.to_string();
             config.imap_pw = imap_pw.to_string();
-            config.certificate_checks = lp.imap_certificate_checks;
+            let provider = get_provider_info(&lp.addr);
+            config.strict_tls = match lp.imap_certificate_checks {
+                CertificateChecks::Automatic => {
+                    provider.map_or(false, |provider| provider.strict_tls)
+                }
+                CertificateChecks::Strict => true,
+                CertificateChecks::AcceptInvalidCertificates
+                | CertificateChecks::AcceptInvalidCertificates2 => false,
+            };
             config.server_flags = server_flags;
         }
 
@@ -719,9 +755,17 @@ impl Imap {
         folder: S,
         server_uids: &[u32],
     ) -> (Option<u32>, usize) {
-        if server_uids.is_empty() {
-            return (None, 0);
-        }
+        let set = match server_uids {
+            [] => return (None, 0),
+            [server_uid] => server_uid.to_string(),
+            [first_uid, .., last_uid] => {
+                // XXX: it is assumed that UIDs are sorted and
+                // contiguous. If UIDs are not contiguous, more
+                // messages than needed will be downloaded.
+                debug_assert!(first_uid < last_uid, "uids must be sorted");
+                format!("{}:{}", first_uid, last_uid)
+            }
+        };
 
         if !self.is_connected() {
             warn!(context, "Not connected");
@@ -737,15 +781,6 @@ impl Imap {
 
         let session = self.session.as_mut().unwrap();
 
-        let set = if server_uids.len() == 1 {
-            server_uids[0].to_string()
-        } else {
-            let first_uid = server_uids[0];
-            let last_uid = server_uids[server_uids.len() - 1];
-            assert!(first_uid < last_uid, "uids must be sorted");
-            format!("{}:{}", first_uid, last_uid)
-        };
-
         let mut msgs = match session.uid_fetch(&set, BODY_FLAGS).await {
             Ok(msgs) => msgs,
             Err(err) => {
@@ -769,7 +804,6 @@ impl Imap {
         let mut last_uid = None;
         let mut count = 0;
 
-        let mut tasks = Vec::with_capacity(server_uids.len());
         while let Some(Ok(msg)) = msgs.next().await {
             let server_uid = msg.uid.unwrap_or_default();
 
@@ -789,32 +823,17 @@ impl Imap {
             let context = context.clone();
             let folder = folder.clone();
 
-            let task = async_std::task::spawn(async move {
-                // safe, as we checked above that there is a body.
-                let body = msg.body().unwrap();
-                let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
+            // safe, as we checked above that there is a body.
+            let body = msg.body().unwrap();
+            let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
-                match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
-                    Ok(_) => Some(server_uid),
-                    Err(err) => {
-                        warn!(context, "dc_receive_imf error: {}", err);
-                        read_errors += 1;
-                        None
-                    }
-                }
-            });
-            tasks.push(task);
-        }
-
-        for task in futures::future::join_all(tasks).await {
-            match task {
-                Some(uid) => {
-                    last_uid = Some(uid);
-                }
-                None => {
+            match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
+                Ok(_) => last_uid = Some(server_uid),
+                Err(err) => {
+                    warn!(context, "dc_receive_imf error: {}", err);
                     read_errors += 1;
                 }
-            }
+            };
         }
 
         if count != server_uids.len() {
@@ -972,7 +991,7 @@ impl Imap {
         uid: u32,
     ) -> Option<ImapActionResult> {
         if uid == 0 {
-            return Some(ImapActionResult::Failed);
+            return Some(ImapActionResult::RetryLater);
         }
         if !self.is_connected() {
             // currently jobs are only performed on the INBOX thread
@@ -1140,54 +1159,54 @@ impl Imap {
                 }
             };
 
+            let mut delimiter = ".".to_string();
+            let mut delimiter_is_default = true;
             let mut sentbox_folder = None;
             let mut mvbox_folder = None;
-
-            let mut delimiter = ".".to_string();
-            if let Some(folder) = folders.next().await {
-                let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
-                if let Some(d) = folder.delimiter() {
-                    if !d.is_empty() {
-                        delimiter = d.to_string();
-                    }
-                }
-            }
-            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
-            let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
+            let mut fallback_folder = get_fallback_folder(&delimiter);
 
             while let Some(folder) = folders.next().await {
                 let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
                 info!(context, "Scanning folder: {:?}", folder);
 
-                if mvbox_folder.is_none()
-                    && (folder.name() == "DeltaChat" || folder.name() == fallback_folder)
-                {
-                    mvbox_folder = Some(folder.name().to_string());
-                }
-
-                if sentbox_folder.is_none() {
-                    if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
-                        sentbox_folder = Some(folder);
-                    } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
-                        sentbox_folder = Some(folder);
+                // Update the delimiter iff there is a different one, but only once.
+                if let Some(d) = folder.delimiter() {
+                    if delimiter_is_default && !d.is_empty() && delimiter != d {
+                        delimiter = d.to_string();
+                        fallback_folder = get_fallback_folder(&delimiter);
+                        delimiter_is_default = false;
                     }
                 }
 
-                if mvbox_folder.is_some() && sentbox_folder.is_some() {
-                    break;
+                if folder.name() == "DeltaChat" {
+                    // Always takes precendent
+                    mvbox_folder = Some(folder.name().to_string());
+                } else if folder.name() == fallback_folder {
+                    // only set iff none has been already set
+                    if mvbox_folder.is_none() {
+                        mvbox_folder = Some(folder.name().to_string());
+                    }
+                } else if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
+                    // Always takes precedent
+                    sentbox_folder = Some(folder.name().to_string());
+                } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
+                    // only set iff none has been already set
+                    if sentbox_folder.is_none() {
+                        sentbox_folder = Some(folder.name().to_string());
+                    }
                 }
             }
-            info!(context, "sentbox folder is {:?}", sentbox_folder);
-
             drop(folders);
 
+            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
+            info!(context, "sentbox folder is {:?}", sentbox_folder);
+
             if mvbox_folder.is_none() && create_mvbox {
                 info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
 
                 match session.create("DeltaChat").await {
                     Ok(_) => {
                         mvbox_folder = Some("DeltaChat".into());
-
                         info!(context, "MVBOX-folder created.",);
                     }
                     Err(err) => {
@@ -1221,23 +1240,16 @@ impl Imap {
                 }
             }
             context
-                .sql
-                .set_raw_config(context, "configured_inbox_folder", Some("INBOX"))
+                .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
                 .await?;
             if let Some(ref mvbox_folder) = mvbox_folder {
                 context
-                    .sql
-                    .set_raw_config(context, "configured_mvbox_folder", Some(mvbox_folder))
+                    .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
                     .await?;
             }
             if let Some(ref sentbox_folder) = sentbox_folder {
                 context
-                    .sql
-                    .set_raw_config(
-                        context,
-                        "configured_sentbox_folder",
-                        Some(sentbox_folder.name()),
-                    )
+                    .set_config(Config::ConfiguredSentboxFolder, Some(sentbox_folder))
                     .await?;
             }
             context
@@ -1354,14 +1366,26 @@ async fn precheck_imf(
             let delete_server_after = context.get_config_delete_server_after().await;
 
             if delete_server_after != Some(0) {
-                context
-                    .do_heuristics_moves(server_folder.as_ref(), msg_id)
+                if msg_id
+                    .needs_move(context, server_folder)
+                    .await
+                    .unwrap_or_default()
+                {
+                    // If the bcc-self message is not moved, directly
+                    // add MarkSeen job, otherwise MarkSeen job is
+                    // added after the Move Job completed.
+                    job::add(
+                        context,
+                        job::Job::new(Action::MoveMsg, msg_id.to_u32(), Params::new(), 0),
+                    )
                     .await;
-                job::add(
-                    context,
-                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
-                )
-                .await;
+                } else {
+                    job::add(
+                        context,
+                        job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                    )
+                    .await;
+                }
             }
         } else if old_server_folder != server_folder {
             info!(
@@ -1395,7 +1419,18 @@ async fn precheck_imf(
         }
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
-            update_server_uid(context, &rfc724_mid, server_folder, server_uid).await;
+            update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
+            if let Ok(MessageState::InSeen) = msg_id.get_state(context).await {
+                job::add(
+                    context,
+                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                )
+                .await;
+            };
+            context
+                .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
+                .await;
+            info!(context, "Updating server_uid and interrupting")
         }
         Ok(true)
     } else {
@@ -1439,7 +1474,7 @@ async fn prefetch_is_reply_to_chat_message(
     false
 }
 
-async fn prefetch_should_download(
+pub(crate) async fn prefetch_should_download(
     context: &Context,
     headers: &[mailparse::MailHeader<'_>],
     show_emails: ShowEmails,
@@ -1447,6 +1482,13 @@ async fn prefetch_should_download(
     let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
     let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await;
 
+    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
+        let from = from.to_ascii_lowercase();
+        from.contains("mailer-daemon") || from.contains("mail-daemon")
+    } else {
+        false
+    };
+
     // Autocrypt Setup Message should be shown even if it is from non-chat client.
     let is_autocrypt_setup_message = headers
         .get_header_value(HeaderDef::AutocryptSetupMessage)
@@ -1457,6 +1499,7 @@ async fn prefetch_should_download(
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message
+        || maybe_ndn
         || match show_emails {
             ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
             ShowEmails::AcceptedContacts => {
@@ -1514,3 +1557,7 @@ async fn message_needs_processing(
 
     true
 }
+
+fn get_fallback_folder(delimiter: &str) -> String {
+    format!("INBOX{}DeltaChat", delimiter)
+}

src/imap/select_folder.rs

@@ -27,7 +27,7 @@ impl Imap {
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 
@@ -51,6 +51,14 @@ impl Imap {
         Ok(())
     }
 
+    /// Issues a CLOSE command if selected folder needs expunge.
+    pub(crate) async fn maybe_close_folder(&mut self, context: &Context) -> Result<()> {
+        if self.config.selected_folder_needs_expunge {
+            self.close_folder(context).await?;
+        }
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -76,10 +84,7 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.selected_folder_needs_expunge };
-        if needs_expunge {
-            self.close_folder(context).await?;
-        }
+        self.maybe_close_folder(context).await?;
 
         // select new folder
         if let Some(ref folder) = folder {

src/imex.rs

@@ -178,10 +178,11 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 ///
 /// The `passphrase` must be at least 2 characters long.
 pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
-    ensure!(
-        passphrase.len() >= 2,
-        "Passphrase must be at least 2 chars long."
-    );
+    let passphrase_begin = if let Some(passphrase_begin) = passphrase.get(..2) {
+        passphrase_begin
+    } else {
+        bail!("Passphrase must be at least 2 chars long.");
+    };
     let private_key = SignedSecretKey::load_self(context).await?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
@@ -196,7 +197,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
             "Passphrase-Format: numeric9x4\r\n",
             "Passphrase-Begin: {}"
         ),
-        &passphrase[..2]
+        passphrase_begin
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
@@ -448,27 +449,33 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
     );
 
-    let files = context
+    // Load IDs only for now, without the file contents, to avoid
+    // consuming too much memory.
+    let file_ids = context
         .sql
         .query_map(
-            "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
+            "SELECT id FROM backup_blobs ORDER BY id",
             paramsv![],
-            |row| {
-                let name: String = row.get(0)?;
-                let blob: Vec<u8> = row.get(1)?;
-
-                Ok((name, blob))
-            },
-            |files| {
-                files
-                    .collect::<std::result::Result<Vec<_>, _>>()
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
                     .map_err(Into::into)
             },
         )
         .await?;
 
     let mut all_files_extracted = true;
-    for (processed_files_cnt, (file_name, file_blob)) in files.into_iter().enumerate() {
+    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        // Load a single blob into memory
+        let (file_name, file_blob) = context
+            .sql
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?)),
+            )
+            .await?;
+
         if context.shall_stop_ongoing().await {
             all_files_extracted = false;
             break;
@@ -776,9 +783,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
@@ -795,12 +802,12 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
@@ -808,7 +815,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_setup_code() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let setupcode = create_setup_code(&t.ctx);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
@@ -823,7 +830,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_export_key_to_asc_file() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)

src/job.rs

@@ -21,6 +21,7 @@ use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::load_imap_deletion_msgid;
 use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::Event;
 use crate::imap::*;
@@ -31,7 +32,7 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
-use crate::sql;
+use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
@@ -94,7 +95,6 @@ pub enum Action {
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
     EmptyServer = 107,
-    OldDeleteMsgOnImap = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -123,7 +123,6 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
-            OldDeleteMsgOnImap => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
             EmptyServer => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
@@ -190,7 +189,7 @@ impl Job {
     /// Saves the job to the database, creating a new entry if necessary.
     ///
     /// The Job is consumed by this method.
-    pub async fn save(self, context: &Context) -> Result<()> {
+    pub(crate) async fn save(self, context: &Context) -> Result<()> {
         let thread: Thread = self.action.into();
 
         info!(context, "saving job for {}-thread: {:?}", thread, self);
@@ -254,40 +253,48 @@ impl Job {
 
                 let res = match err {
                     async_smtp::smtp::error::Error::Permanent(ref response) => {
-                        match response.code {
+                        // Workaround for incorrectly configured servers returning permanent errors
+                        // instead of temporary ones.
+                        let maybe_transient = match response.code {
                             // Sometimes servers send a permanent error when actually it is a temporary error
                             // For documentation see https://tools.ietf.org/html/rfc3463
-
-                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
                             Code {
                                 category: Category::MailSystem,
                                 detail: Detail::Zero,
                                 ..
-                            } => Status::RetryLater,
-
-                            _ => {
-                                // If we do not retry, add an info message to the chat
-                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
-                                match Message::load_from_db(context, MsgId::new(self.foreign_id))
-                                    .await
-                                {
-                                    Ok(message) => {
-                                        chat::add_info_msg(
-                                            context,
-                                            message.chat_id,
-                                            err.to_string(),
-                                        )
-                                        .await
-                                    }
-                                    Err(e) => warn!(
-                                        context,
-                                        "couldn't load chat_id to inform user about SMTP error: {}",
-                                        e
-                                    ),
-                                };
-
-                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } => {
+                                // Ignore status code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                                // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
+                                // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
+                                //
+                                // Other enhanced status codes, such as Postfix
+                                // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
+                                // are not ignored.
+                                response.message.get(0) == Some(&"5.5.0".to_string())
                             }
+                            _ => false,
+                        };
+
+                        if maybe_transient {
+                            Status::RetryLater
+                        } else {
+                            // If we do not retry, add an info message to the chat.
+                            // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
+                            // should definitely go here, because user has to open the link to
+                            // resume message sending.
+                            let msg_id = MsgId::new(self.foreign_id);
+                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+                            match Message::load_from_db(context, msg_id).await {
+                                Ok(message) => {
+                                    chat::add_info_msg(context, message.chat_id, err.to_string())
+                                        .await
+                                }
+                                Err(e) => error!(
+                                    context,
+                                    "couldn't load chat_id to inform user about SMTP error: {}", e
+                                ),
+                            };
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
                     async_smtp::smtp::error::Error::Transient(_) => {
@@ -329,7 +336,7 @@ impl Job {
         }
     }
 
-    pub async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
         if !smtp.is_connected().await {
             let loginparam = LoginParam::from_database(context, "configured_").await;
@@ -498,16 +505,13 @@ impl Job {
     }
 
     async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-
-        if let Err(err) = imap.ensure_configured_folders(context, true).await {
-            warn!(context, "could not configure folders: {:?}", err);
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
-        let dest_folder = context
-            .sql
-            .get_raw_config(context, "configured_mvbox_folder")
-            .await;
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
 
         if let Some(dest_folder) = dest_folder {
             let server_folder = msg.server_folder.as_ref().unwrap();
@@ -518,7 +522,7 @@ impl Job {
             {
                 ImapActionResult::RetryLater => Status::RetryLater,
                 ImapActionResult::Success => {
-                    // XXX Rust-Imap provides no target uid on mv, so just set it to 0
+                    // Rust-Imap provides no target uid on mv, so just set it to 0, update again when precheck_imf() is called for the moved message
                     message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await;
                     Status::Finished(Ok(()))
                 }
@@ -541,6 +545,11 @@ impl Job {
     /// records pointing to the same message on the server, the job
     /// also removes the message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         if !msg.rfc724_mid.is_empty() {
@@ -611,26 +620,48 @@ impl Job {
     }
 
     async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = context
-                .sql
-                .get_raw_config(context, "configured_mvbox_folder")
-                .await
-            {
+            if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
                 imap.empty_folder(context, &mvbox_folder).await;
             }
         }
         if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            imap.empty_folder(context, "INBOX").await;
+            if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
+                imap.empty_folder(context, &inbox_folder).await;
+            }
         }
         Status::Finished(Ok(()))
     }
 
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
-        match imap.set_seen(context, folder, msg.server_uid).await {
+
+        let result = if msg.server_uid == 0 {
+            // The message is moved or deleted by us.
+            //
+            // Do not call set_seen with zero UID, as it will return
+            // ImapActionResult::RetryLater, but we do not want to
+            // retry. If the message was moved, we will create another
+            // job to mark the message as seen later. If it was
+            // deleted, there is nothing to do.
+            ImapActionResult::Failed
+        } else {
+            imap.set_seen(context, folder, msg.server_uid).await
+        };
+
+        match result {
             ImapActionResult::RetryLater => Status::RetryLater,
             ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
             ImapActionResult::Success | ImapActionResult::Failed => {
@@ -798,7 +829,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
 
     if rendered_msg.is_encrypted && !needs_encryption {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
-        msg.save_param_to_disk(context).await;
+        msg.update_param(context).await;
     }
 
     ensure!(!recipients.is_empty(), "no recipients for smtp job set");
@@ -815,31 +846,11 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     Ok(Some(job))
 }
 
-#[derive(Debug)]
-pub enum Connection<'a> {
+pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
-    if let Some(delete_server_after) = context.get_config_delete_server_after().await {
-        let threshold_timestamp = time() - delete_server_after;
-
-        context
-            .sql
-            .query_row_optional(
-                "SELECT id FROM msgs \
-             WHERE timestamp < ? \
-             AND server_uid != 0",
-                paramsv![threshold_timestamp],
-                |row| row.get::<_, MsgId>(0),
-            )
-            .await
-    } else {
-        Ok(None)
-    }
-}
-
 async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         Some(Job::new(
@@ -963,7 +974,6 @@ async fn perform_job_action(
             location::job_maybe_send_locations_ended(context, job).await
         }
         Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
-        Action::OldDeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
@@ -973,10 +983,7 @@ async fn perform_job_action(
         }
     };
 
-    info!(
-        context,
-        "Inbox finished immediate try {} of job {}", tries, job
-    );
+    info!(context, "Finished immediate try {} of job {}", tries, job);
 
     try_res
 }
@@ -1024,19 +1031,22 @@ pub async fn add(context: &Context, job: Job) {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
             | Action::EmptyServer
-            | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
             | Action::MarkseenMsgOnImap
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
-                context.interrupt_inbox(false).await;
+                context
+                    .interrupt_inbox(InterruptInfo::new(false, None))
+                    .await;
             }
             Action::MaybeSendLocations
             | Action::MaybeSendLocationsEnded
             | Action::SendMdn
             | Action::SendMsgToSmtp => {
                 info!(context, "interrupt: smtp");
-                context.interrupt_smtp(false).await;
+                context
+                    .interrupt_smtp(InterruptInfo::new(false, None))
+                    .await;
             }
         }
     }
@@ -1051,38 +1061,49 @@ pub async fn add(context: &Context, job: Job) {
 pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
-    probe_network: bool,
+    info: &InterruptInfo,
 ) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
-    let query = if !probe_network {
+
+    let query;
+    let params;
+    let t = time();
+    let m;
+    let thread_i = thread as i64;
+
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
+SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+FROM jobs
+WHERE thread=? AND foreign_id=?
+ORDER BY action DESC, added_timestamp
+LIMIT 1;
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
         // processing for first-try and after backoff-timeouts:
         // process jobs in the order they were added.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND desired_timestamp<=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#
+"#;
+        params = paramsv![thread_i, t];
     } else {
         // processing after call to dc_maybe_network():
         // process _all_ pending jobs that failed before
         // in the order of their backoff-times.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND tries>0
 ORDER BY desired_timestamp, action DESC
 LIMIT 1;
-"#
-    };
-
-    let thread_i = thread as i64;
-    let t = time();
-    let params = if !probe_network {
-        paramsv![thread_i, t]
-    } else {
-        paramsv![thread_i]
+"#;
+        params = paramsv![thread_i];
     };
 
     let job = loop {
@@ -1187,23 +1208,38 @@ mod tests {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_none());
 
         insert_job(&t.ctx, 1).await;
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_some());
     }
 
     #[async_std::test]
     async fn test_load_next_job_one() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         insert_job(&t.ctx, 1).await;
 
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_some());
     }
 }

src/key.rs

@@ -9,6 +9,7 @@ use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::*;
@@ -106,7 +107,7 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
 
     /// The fingerprint for the key.
     fn fingerprint(&self) -> Fingerprint {
-        Fingerprint::new(KeyTrait::fingerprint(self))
+        Fingerprint::new(KeyTrait::fingerprint(self)).expect("Invalid fingerprint from rpgp")
     }
 }
 
@@ -246,7 +247,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
         Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
+            let start = std::time::SystemTime::now();
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
@@ -257,7 +258,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
+                start.elapsed().unwrap_or_default().as_secs()
             );
             Ok(keypair)
         }
@@ -354,12 +355,15 @@ pub async fn store_self_keypair(
 }
 
 /// A key fingerprint
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
-    pub fn new(v: Vec<u8>) -> Fingerprint {
-        Fingerprint(v)
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+        match v.len() {
+            20 => Ok(Fingerprint(v)),
+            _ => Err(FingerprintError::WrongLength),
+        }
     }
 
     /// Make a hex string from the fingerprint.
@@ -389,42 +393,26 @@ impl fmt::Display for Fingerprint {
 
 /// Parse a human-readable or otherwise formatted fingerprint.
 impl std::str::FromStr for Fingerprint {
-    type Err = hex::FromHexError;
+    type Err = FingerprintError;
 
     fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
         let hex_repr: String = input
+            .to_uppercase()
             .chars()
             .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
             .collect();
         let v: Vec<u8> = hex::decode(hex_repr)?;
-        Ok(Fingerprint(v))
+        let fp = Fingerprint::new(v)?;
+        Ok(fp)
     }
 }
 
-/// Bring a human-readable or otherwise formatted fingerprint back to the 40-characters-uppercase-hex format.
-pub fn dc_normalize_fingerprint(fp: &str) -> String {
-    fp.to_uppercase()
-        .chars()
-        .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
-        .collect()
-}
-
-/// Make a fingerprint human-readable, in hex format.
-pub fn dc_format_fingerprint(fingerprint: &str) -> String {
-    // split key into chunks of 4 with space, and 20 newline
-    let mut res = String::new();
-
-    for (i, c) in fingerprint.chars().enumerate() {
-        if i > 0 && i % 20 == 0 {
-            res += "\n";
-        } else if i > 0 && i % 4 == 0 {
-            res += " ";
-        }
-
-        res += &c.to_string();
-    }
-
-    res
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
 }
 
 #[cfg(test)]
@@ -432,6 +420,8 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
 
+    use std::error::Error;
+
     use async_std::sync::Arc;
     use lazy_static::lazy_static;
 
@@ -439,23 +429,6 @@ mod tests {
         static ref KEYPAIR: KeyPair = alice_keypair();
     }
 
-    #[test]
-    fn test_normalize_fingerprint() {
-        let fingerprint = dc_normalize_fingerprint(" 1234  567890 \n AbcD abcdef ABCDEF ");
-
-        assert_eq!(fingerprint, "1234567890ABCDABCDEFABCDEF");
-    }
-
-    #[test]
-    fn test_format_fingerprint() {
-        let fingerprint = dc_format_fingerprint("1234567890ABCDABCDEFABCDEF1234567890ABCD");
-
-        assert_eq!(
-            fingerprint,
-            "1234 5678 90AB CDAB CDEF\nABCD EF12 3456 7890 ABCD"
-        );
-    }
-
     #[test]
     fn test_from_armored_string() {
         let (private_key, _) = SignedSecretKey::from_asc(
@@ -577,8 +550,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = dummy_context().await;
-        configure_alice_keypair(&t.ctx).await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
         assert_eq!(alice.public, pubkey);
         let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
@@ -586,9 +559,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_public() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -598,9 +570,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_secret() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -610,11 +581,10 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -641,7 +611,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let ctx = Arc::new(t.ctx);
 
         let ctx1 = ctx.clone();
@@ -685,32 +655,46 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 
     #[test]
     fn test_fingerprint_from_str() {
-        let res = Fingerprint::new(vec![1, 2, 4, 8, 16, 32, 64, 128, 255]);
+        let res = Fingerprint::new(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
 
-        let fp: Fingerprint = "0102040810204080FF".parse().unwrap();
+        let fp: Fingerprint = "0102030405060708090A0B0c0d0e0F1011121314".parse().unwrap();
         assert_eq!(fp, res);
 
-        let fp: Fingerprint = "zzzz 0102 0408\n1020 4080 FF zzz".parse().unwrap();
+        let fp: Fingerprint = "zzzz 0102 0304 0506\n0708090a0b0c0D0E0F1011121314 yyy"
+            .parse()
+            .unwrap();
         assert_eq!(fp, res);
 
         let err = "1".parse::<Fingerprint>().err().unwrap();
-        assert_eq!(err, hex::FromHexError::OddLength);
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
     }
 
     #[test]
     fn test_fingerprint_hex() {
-        let fp = Fingerprint::new(vec![1, 2, 4, 8, 16, 32, 64, 128, 255]);
-        assert_eq!(fp.hex(), "0102040810204080FF");
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(fp.hex(), "0102040810204080FF0A0B0C0D0E0F1011121314");
     }
 
     #[test]
     fn test_fingerprint_to_string() {
         let fp = Fingerprint::new(vec![
-            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255,
-        ]);
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255, 19, 20,
+        ])
+        .unwrap();
         assert_eq!(
             fp.to_string(),
-            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF"
+            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF 1314"
         );
     }
 }

src/keyring.rs

@@ -79,8 +79,8 @@ mod tests {
     #[async_std::test]
     async fn test_keyring_load_self() {
         // new_self() implies load_self()
-        let t = dummy_context().await;
-        configure_alice_keypair(&t.ctx).await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let alice = alice_keypair();
 
         let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();

src/lib.rs

@@ -1,6 +1,11 @@
 #![forbid(unsafe_code)]
-#![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![allow(clippy::match_bool)]
+#![deny(
+    clippy::correctness,
+    missing_debug_implementations,
+    clippy::all,
+    clippy::indexing_slicing
+)]
+#![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
 #[macro_use]
 extern crate num_derive;
@@ -11,8 +16,6 @@ extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 
@@ -45,6 +48,7 @@ pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
+pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;

src/location.rs

@@ -530,7 +530,7 @@ pub async fn save(
             accuracy,
             ..
         } = location;
-        context
+        let (loc_id, ts) = context
             .sql
             .with_conn(move |mut conn| {
                 let mut stmt_test = conn
@@ -569,9 +569,11 @@ pub async fn save(
                         )?;
                     }
                 }
-                Ok(())
+                Ok((newest_location_id, newest_timestamp))
             })
             .await?;
+        newest_timestamp = ts;
+        newest_location_id = loc_id;
     }
 
     Ok(newest_location_id)
@@ -722,11 +724,11 @@ pub(crate) async fn job_maybe_send_locations_ended(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_kml_parse() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
 
         let xml =
             b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";

src/log.rs

@@ -41,6 +41,17 @@ macro_rules! error {
     }};
 }
 
+#[macro_export]
+macro_rules! error_network {
+    ($ctx:expr, $msg:expr) => {
+        error_network!($ctx, $msg,)
+    };
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+        let formatted = format!($msg, $($args),*);
+        emit_event!($ctx, $crate::Event::ErrorNetwork(formatted));
+    }};
+}
+
 #[macro_export]
 macro_rules! emit_event {
     ($ctx:expr, $event:expr) => {

src/login_param.rs

@@ -130,10 +130,6 @@ impl LoginParam {
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
     pub async fn save_to_database(
         &self,
@@ -277,21 +273,15 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-    match certificate_checks {
-        CertificateChecks::Automatic => {
-            // Same as AcceptInvalidCertificates for now.
-            // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
-        }
-        CertificateChecks::Strict => tls_builder,
-        CertificateChecks::AcceptInvalidCertificates
-        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
+
+    if strict_tls {
+        tls_builder
+    } else {
+        tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
+            .danger_accept_invalid_certs(true)
     }
 }
 

src/lot.rs

@@ -1,5 +1,7 @@
 use deltachat_derive::{FromSql, ToSql};
 
+use crate::key::Fingerprint;
+
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
 /// Lot objects are created
@@ -14,7 +16,7 @@ pub struct Lot {
     pub(crate) timestamp: i64,
     pub(crate) state: LotState,
     pub(crate) id: u32,
-    pub(crate) fingerprint: Option<String>,
+    pub(crate) fingerprint: Option<Fingerprint>,
     pub(crate) invitenumber: Option<String>,
     pub(crate) auth: Option<String>,
 }

src/message.rs

@@ -6,6 +6,7 @@ use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
@@ -14,7 +15,7 @@ use crate::error::{ensure, Error};
 use crate::events::Event;
 use crate::job::{self, Action};
 use crate::lot::{Lot, LotState, Meaning};
-use crate::mimeparser::SystemMessage;
+use crate::mimeparser::{FailureReport, SystemMessage};
 use crate::param::*;
 use crate::pgp::*;
 use crate::stock::StockMessage;
@@ -68,18 +69,38 @@ impl MsgId {
         self.0 == 0
     }
 
-    /// Whether the message ID is the special marker1 marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_marker1(self) -> bool {
-        self.0 == DC_MSG_ID_MARKER1
+    /// Returns message state.
+    pub async fn get_state(self, context: &Context) -> crate::sql::Result<MessageState> {
+        let result = context
+            .sql
+            .query_get_value_result("SELECT state FROM msgs WHERE id=?", paramsv![self])
+            .await?
+            .unwrap_or_default();
+        Ok(result)
     }
 
-    /// Whether the message ID is the special day marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_daymarker(self) -> bool {
-        self.0 == DC_MSG_ID_DAYMARKER
+    /// Returns true if the message needs to be moved from `folder`.
+    pub async fn needs_move(self, context: &Context, folder: &str) -> Result<bool, Error> {
+        if !context.get_config_bool(Config::MvboxMove).await {
+            return Ok(false);
+        }
+
+        if context.is_mvbox(folder).await {
+            return Ok(false);
+        }
+
+        let msg = Message::load_from_db(context, self).await?;
+
+        if msg.is_setupmessage() {
+            // do not move setup messages;
+            // there may be a non-delta device that wants to handle it
+            return Ok(false);
+        }
+
+        match msg.is_dc_message {
+            MessengerMessage::No => Ok(false),
+            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
+        }
     }
 
     /// Put message into trash chat and delete message text.
@@ -143,16 +164,7 @@ impl MsgId {
 
 impl std::fmt::Display for MsgId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // Would be nice if we could use match here, but no computed values in ranges.
-        if self.0 == DC_MSG_ID_MARKER1 {
-            write!(f, "Msg#Marker1")
-        } else if self.0 == DC_MSG_ID_DAYMARKER {
-            write!(f, "Msg#DayMarker")
-        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
-            write!(f, "Msg#UnknownSpecial")
-        } else {
-            write!(f, "Msg#{}", self.0)
-        }
+        write!(f, "Msg#{}", self.0)
     }
 }
 
@@ -246,6 +258,8 @@ pub struct Message {
     pub(crate) timestamp_sort: i64,
     pub(crate) timestamp_sent: i64,
     pub(crate) timestamp_rcvd: i64,
+    pub(crate) ephemeral_timer: u32,
+    pub(crate) ephemeral_timestamp: i64,
     pub(crate) text: Option<String>,
     pub(crate) rfc724_mid: String,
     pub(crate) in_reply_to: Option<String>,
@@ -255,6 +269,7 @@ pub struct Message {
     pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
+    pub(crate) error: String,
     pub(crate) param: Params,
 }
 
@@ -287,8 +302,11 @@ impl Message {
                     "    m.timestamp AS timestamp,",
                     "    m.timestamp_sent AS timestamp_sent,",
                     "    m.timestamp_rcvd AS timestamp_rcvd,",
+                    "    m.ephemeral_timer AS ephemeral_timer,",
+                    "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
+                    "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.txt AS txt,",
                     "    m.param AS param,",
@@ -314,8 +332,11 @@ impl Message {
                     msg.timestamp_sort = row.get("timestamp")?;
                     msg.timestamp_sent = row.get("timestamp_sent")?;
                     msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                    msg.ephemeral_timer = row.get("ephemeral_timer")?;
+                    msg.ephemeral_timestamp = row.get("ephemeral_timestamp")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
+                    msg.error = row.get("error")?;
                     msg.is_dc_message = row.get("msgrmsg")?;
 
                     let text;
@@ -390,7 +411,7 @@ impl Message {
                     }
 
                     if !self.id.is_unset() {
-                        self.save_param_to_disk(context).await;
+                        self.update_param(context).await;
                     }
                 }
             }
@@ -507,6 +528,14 @@ impl Message {
         self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
+    pub fn get_ephemeral_timer(&self) -> u32 {
+        self.ephemeral_timer
+    }
+
+    pub fn get_ephemeral_timestamp(&self) -> i64 {
+        self.ephemeral_timestamp
+    }
+
     pub async fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
         let mut ret = Lot::new();
 
@@ -609,6 +638,39 @@ impl Message {
         None
     }
 
+    /// split a webrtc_instance as defined by the corresponding config-value into a type and a url
+    pub fn parse_webrtc_instance(instance: &str) -> (VideochatType, String) {
+        let mut split = instance.splitn(2, ':');
+        let type_str = split.next().unwrap_or_default().to_lowercase();
+        let url = split.next();
+        if type_str == "basicwebrtc" {
+            (
+                VideochatType::BasicWebrtc,
+                url.unwrap_or_default().to_string(),
+            )
+        } else {
+            (VideochatType::Unknown, instance.to_string())
+        }
+    }
+
+    pub fn get_videochat_url(&self) -> Option<String> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).1);
+            }
+        }
+        None
+    }
+
+    pub fn get_videochat_type(&self) -> Option<VideochatType> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).0);
+            }
+        }
+        None
+    }
+
     pub fn set_text(&mut self, text: Option<String>) {
         self.text = text;
     }
@@ -643,10 +705,10 @@ impl Message {
         if duration > 0 {
             self.param.set_int(Param::Duration, duration);
         }
-        self.save_param_to_disk(context).await;
+        self.update_param(context).await;
     }
 
-    pub async fn save_param_to_disk(&mut self, context: &Context) -> bool {
+    pub async fn update_param(&mut self, context: &Context) -> bool {
         context
             .sql
             .execute(
@@ -763,9 +825,10 @@ impl From<MessageState> for LotState {
 impl MessageState {
     pub fn can_fail(self) -> bool {
         match self {
-            MessageState::OutPreparing | MessageState::OutPending | MessageState::OutDelivered => {
-                true
-            }
+            MessageState::OutPreparing
+            | MessageState::OutPending
+            | MessageState::OutDelivered
+            | MessageState::OutMdnRcvd => true, // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
             _ => false,
         }
     }
@@ -887,6 +950,17 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         ret += "\n";
     }
 
+    if msg.ephemeral_timer != 0 {
+        ret += &format!("Ephemeral timer: {}\n", msg.ephemeral_timer);
+    }
+
+    if msg.ephemeral_timestamp != 0 {
+        ret += &format!(
+            "Expires: {}\n",
+            dc_timestamp_to_str(msg.ephemeral_timestamp)
+        );
+    }
+
     if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
@@ -937,8 +1011,9 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
     }
 
     ret += "\n";
-    if let Some(err) = msg.param.get(Param::Error) {
-        ret += &format!("Error: {}", err)
+
+    if !msg.error.is_empty() {
+        ret += &format!("Error: {}", msg.error);
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -1091,6 +1166,14 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
     let mut send_event = false;
 
     for (id, curr_state, curr_blocked) in msgs.into_iter() {
+        if let Err(err) = id.start_ephemeral_timer(context).await {
+            error!(
+                context,
+                "Failed to start ephemeral timer for message {}: {}", id, err
+            );
+            continue;
+        }
+
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, id, MessageState::InSeen).await;
@@ -1147,7 +1230,7 @@ pub async fn star_msgs(context: &Context, msg_ids: Vec<MsgId>, star: bool) -> bo
         .is_ok()
 }
 
-/// Returns a summary test.
+/// Returns a summary text.
 pub async fn get_summarytext_by_raw(
     viewtype: Viewtype,
     text: Option<impl AsRef<str>>,
@@ -1191,6 +1274,13 @@ pub async fn get_summarytext_by_raw(
                 format!("{} – {}", label, file_name)
             }
         }
+        Viewtype::VideochatInvitation => {
+            append_text = false;
+            context
+                .stock_str(StockMessage::VideochatInvitation)
+                .await
+                .into_owned()
+        }
         _ => {
             if param.get_cmd() != SystemMessage::LocationOnly {
                 "".to_string()
@@ -1251,39 +1341,44 @@ pub async fn exists(context: &Context, msg_id: MsgId) -> bool {
 
 pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id).await {
+        let error = error.map(|e| e.as_ref().to_string()).unwrap_or_default();
         if msg.state.can_fail() {
             msg.state = MessageState::OutFailed;
-        }
-        if let Some(error) = error {
-            msg.param.set(Param::Error, error.as_ref());
-            warn!(context, "Message failed: {}", error.as_ref());
+            warn!(context, "{} failed: {}", msg_id, error);
+        } else {
+            warn!(
+                context,
+                "{} seems to have failed ({}), but state is {}", msg_id, error, msg.state
+            )
         }
 
-        if context
+        match context
             .sql
             .execute(
-                "UPDATE msgs SET state=?, param=? WHERE id=?;",
-                paramsv![msg.state, msg.param.to_string(), msg_id],
+                "UPDATE msgs SET state=?, error=? WHERE id=?;",
+                paramsv![msg.state, error, msg_id],
             )
             .await
-            .is_ok()
         {
-            context.emit_event(Event::MsgFailed {
+            Ok(_) => context.emit_event(Event::MsgFailed {
                 chat_id: msg.chat_id,
                 msg_id,
-            });
+            }),
+            Err(e) => {
+                warn!(context, "{:?}", e);
+            }
         }
     }
 }
 
 /// returns Some if an event should be send
-pub async fn mdn_from_ext(
+pub async fn handle_mdn(
     context: &Context,
     from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Option<(ChatId, MsgId)> {
-    if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
+    if from_id <= DC_CONTACT_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
         return None;
     }
 
@@ -1318,10 +1413,10 @@ pub async fn mdn_from_ext(
     if let Ok((msg_id, chat_id, chat_type, msg_state)) = res {
         let mut read_by_all = false;
 
-        // if already marked as MDNS_RCVD msgstate_can_fail() returns false.
-        // however, it is important, that ret_msg_id is set above as this
-        // will allow the caller eg. to move the message away
-        if msg_state.can_fail() {
+        if msg_state == MessageState::OutPreparing
+            || msg_state == MessageState::OutPending
+            || msg_state == MessageState::OutDelivered
+        {
             let mdn_already_in_table = context
                 .sql
                 .exists(
@@ -1384,6 +1479,69 @@ pub async fn mdn_from_ext(
     None
 }
 
+/// Marks a message as failed after an ndn (non-delivery-notification) arrived.
+/// Where appropriate, also adds an info message telling the user which of the recipients of a group message failed.
+pub(crate) async fn handle_ndn(
+    context: &Context,
+    failed: &FailureReport,
+    error: Option<impl AsRef<str>>,
+) {
+    if failed.rfc724_mid.is_empty() {
+        return;
+    }
+
+    let res = context
+        .sql
+        .query_row(
+            concat!(
+                "SELECT",
+                "    m.id AS msg_id,",
+                "    c.id AS chat_id,",
+                "    c.type AS type",
+                " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+                " WHERE rfc724_mid=? AND from_id=1",
+            ),
+            paramsv![failed.rfc724_mid],
+            |row| {
+                Ok((
+                    row.get::<_, MsgId>("msg_id")?,
+                    row.get::<_, ChatId>("chat_id")?,
+                    row.get::<_, Chattype>("type")?,
+                ))
+            },
+        )
+        .await;
+    if let Err(ref err) = res {
+        info!(context, "Failed to select NDN {:?}", err);
+    }
+
+    if let Ok((msg_id, chat_id, chat_type)) = res {
+        set_msg_failed(context, msg_id, error).await;
+
+        if chat_type == Chattype::Group || chat_type == Chattype::VerifiedGroup {
+            if let Some(failed_recipient) = &failed.failed_recipient {
+                let contact_id =
+                    Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
+                if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+                    // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
+                    chat::add_info_msg(
+                        context,
+                        chat_id,
+                        context
+                            .stock_string_repl_str(
+                                StockMessage::FailedSendingTo,
+                                contact.get_display_name(),
+                            )
+                            .await,
+                    )
+                    .await;
+                    context.emit_event(Event::ChatModified(chat_id));
+                }
+            }
+        }
+    }
+}
+
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
 pub async fn get_real_msg_cnt(context: &Context) -> i32 {
     match context
@@ -1572,7 +1730,7 @@ mod tests {
     async fn test_prepare_message_and_send() {
         use crate::config::Config;
 
-        let d = test::dummy_context().await;
+        let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
         let contact = Contact::create(ctx, "", "dest@example.com")
@@ -1596,7 +1754,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_summarytext_by_raw() {
-        let d = test::dummy_context().await;
+        let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
         let some_text = Some("bla bla".to_string());
@@ -1681,4 +1839,19 @@ mod tests {
             "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
         );
     }
+
+    #[async_std::test]
+    async fn test_parse_webrtc_instance() {
+        let (webrtc_type, url) = Message::parse_webrtc_instance("basicwebrtc:https://foo/bar");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "https://foo/bar");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("bAsIcwEbrTc:url");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "url");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("https://foo/bar?key=val#key=val");
+        assert_eq!(webrtc_type, VideochatType::Unknown);
+        assert_eq!(url, "https://foo/bar?key=val#key=val");
+    }
 }

src/mimefactory.rs

@@ -9,6 +9,7 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
 use crate::location;
 use crate::message::{self, Message};
@@ -351,16 +352,47 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    let raw = message::get_summarytext_by_raw(
-                        self.msg.viewtype,
-                        self.msg.text.as_ref(),
-                        &self.msg.param,
-                        32,
-                        self.context,
-                    )
-                    .await;
-                    let raw_subject = raw.lines().next().unwrap_or_default();
-                    format!("Chat: {}", raw_subject)
+                    match chat.param.get(Param::LastSubject) {
+                        Some(last_subject) => {
+                            let subject_start = if last_subject.starts_with("Chat:") {
+                                0
+                            } else {
+                                // "Antw:" is the longest abbreviation in
+                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+                                // so look at the first _5_ characters:
+                                match last_subject.chars().take(5).position(|c| c == ':') {
+                                    Some(prefix_end) => prefix_end + 1,
+                                    None => 0,
+                                }
+                            };
+                            format!(
+                                "Re: {}",
+                                last_subject
+                                    .chars()
+                                    .skip(subject_start)
+                                    .collect::<String>()
+                                    .trim()
+                            )
+                        }
+                        None => {
+                            let self_name = match self.context.get_config(Config::Displayname).await
+                            {
+                                Some(name) => name,
+                                None => self
+                                    .context
+                                    .get_config(Config::Addr)
+                                    .await
+                                    .unwrap_or_default(),
+                            };
+
+                            self.context
+                                .stock_string_repl_str(
+                                    StockMessage::SubjectForNewContact,
+                                    self_name,
+                                )
+                                .await
+                        }
+                    }
                 }
             }
             Loaded::MDN { .. } => self
@@ -409,6 +441,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
+        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -465,7 +499,16 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(self.context).await?;
 
-        let subject = encode_words(&subject_str);
+        let subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str
+        } else {
+            encode_words(&subject_str)
+        };
 
         let mut message = match self.loaded {
             Loaded::Message { .. } => {
@@ -493,6 +536,14 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
         };
 
+        let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(self.context).await?;
+        if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
+            protected_headers.push(Header::new(
+                "Ephemeral-Timer".to_string(),
+                duration.to_string(),
+            ));
+        }
+
         // we could also store the message-id in the protected headers
         // which would probably help to survive providers like
         // Outlook.com or hotmail which mangle the Message-ID.
@@ -743,6 +794,26 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     "location-streaming-enabled".into(),
                 ));
             }
+            SystemMessage::EphemeralTimerChanged => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "ephemeral-timer-changed".to_string(),
+                ));
+            }
+            SystemMessage::LocationOnly => {
+                // This should prevent automatic replies,
+                // such as non-delivery reports.
+                //
+                // See https://tools.ietf.org/html/rfc3834
+                //
+                // Adding this header without encryption leaks some
+                // information about the message contents, but it can
+                // already be easily guessed from message timing and size.
+                unprotected_headers.push(Header::new(
+                    "Auto-Submitted".to_string(),
+                    "auto-generated".to_string(),
+                ));
+            }
             SystemMessage::AutocryptSetupMessage => {
                 unprotected_headers
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
@@ -804,6 +875,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if self.msg.viewtype == Viewtype::Sticker {
             protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
+        } else if self.msg.viewtype == Viewtype::VideochatInvitation {
+            protected_headers.push(Header::new(
+                "Chat-Content".into(),
+                "videochat-invitation".into(),
+            ));
+            protected_headers.push(Header::new(
+                "Chat-Webrtc-Room".into(),
+                self.msg
+                    .param
+                    .get(Param::WebrtcRoom)
+                    .unwrap_or_default()
+                    .into(),
+            ));
         }
 
         if self.msg.viewtype == Viewtype::Voice
@@ -1174,6 +1258,10 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chatlist::Chatlist;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::mimeparser::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1181,6 +1269,9 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
+        assert!(!display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1192,6 +1283,25 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
+    #[test]
+    fn test_render_email_address_noescape() {
+        let display_name = "a space";
+        let addr = "x@y.org";
+
+        assert!(display_name.is_ascii());
+        assert!(display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+        let s = format!(
+            "{}",
+            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
+        );
+
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        assert_eq!(s, "a space <x@y.org>");
+    }
+
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1234,4 +1344,241 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
+
+    #[async_std::test]
+    async fn test_subject() {
+        // 1.: Receive a mail from an MUA or Delta Chat
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Infos: 42"
+        );
+
+        // 2. Receive a message from Delta Chat when we did not send any messages before
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        // 3. Send the first message to a new contact
+        let t = TestContext::new_alice().await;
+
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::Displayname, Some("Alice"))
+            .await
+            .unwrap();
+        assert_eq!(first_subject_str(t).await, "Message from Alice");
+
+        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        // 5. Receive an mdn (read receipt) and make sure the mdn's subject is not used
+        let t = TestContext::new_alice().await;
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@example.com\n\
+            To: Charlie <charlie@example.com>\n\
+            Subject: Hello, Charlie\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        let new_msg = incoming_msg_to_reply_msg(b"From: charlie@example.com\n\
+                 To: alice@example.com\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;charlie@example.com\n\
+                 Final-Recipient: rfc822;charlie@example.com\n\
+                 Original-Message-ID: <2893@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n", &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        // The subject string should not be "Re: message opened"
+        assert_eq!("Re: Hello, Charlie", mf.subject_str().await);
+    }
+
+    async fn first_subject_str(t: TestContext) -> String {
+        let contact_id =
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.com", Origin::ManuallyCreated)
+                .await
+                .unwrap()
+                .0;
+
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+
+        mf.subject_str().await
+    }
+
+    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+        let t = TestContext::new_alice().await;
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        mf.subject_str().await
+    }
+
+    // Creates a `Message` that replies "Hi" to the incoming email in `imf_raw`.
+    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+        context
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(context, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        new_msg
+    }
+
+    #[async_std::test]
+    // This test could still be extended
+    async fn test_render_reply() {
+        let t = TestContext::new_alice().await;
+        let context = &t.ctx;
+
+        let msg = incoming_msg_to_reply_msg(
+            b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            context,
+        )
+        .await;
+
+        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+
+        let recipients = mimefactory.recipients();
+        assert_eq!(recipients, vec!["charlie@example.com"]);
+
+        let rendered_msg = mimefactory.render().await.unwrap();
+
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        assert_eq!(
+            mail.headers
+                .iter()
+                .find(|h| h.get_key() == "MIME-Version")
+                .unwrap()
+                .get_value(),
+            "1.0"
+        );
+
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+            .await
+            .unwrap();
+    }
 }

src/mimeparser.rs

@@ -3,6 +3,7 @@ use std::future::Future;
 use std::pin::Pin;
 
 use deltachat_derive::{FromSql, ToSql};
+use lazy_static::lazy_static;
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 
@@ -17,6 +18,7 @@ use crate::e2ee;
 use crate::error::{bail, Result};
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
 use crate::param::*;
@@ -44,7 +46,14 @@ pub struct MimeMessage {
     pub from: Vec<SingleInfo>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
-    pub signatures: HashSet<String>,
+
+    /// Set of valid signature fingerprints if a message is an
+    /// Autocrypt encrypted and signed message.
+    ///
+    /// If a message is not encrypted or the signature is not valid,
+    /// this set is empty.
+    pub signatures: HashSet<Fingerprint>,
+
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
@@ -52,7 +61,8 @@ pub struct MimeMessage {
     pub message_kml: Option<location::Kml>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
-    pub(crate) reports: Vec<Report>,
+    pub(crate) mdn_reports: Vec<Report>,
+    pub(crate) failure_report: Option<FailureReport>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -73,6 +83,9 @@ pub enum SystemMessage {
     SecurejoinMessage = 7,
     LocationStreamingEnabled = 8,
     LocationOnly = 9,
+
+    /// Chat ephemeral message timer is changed.
+    EphemeralTimerChanged = 10,
 }
 
 impl Default for SystemMessage {
@@ -175,14 +188,16 @@ impl MimeMessage {
             signatures,
             gossipped_addr,
             is_forwarded: false,
-            reports: Vec::new(),
+            mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
             user_avatar: None,
             group_avatar: None,
+            failure_report: None,
         };
         parser.parse_mime_recursive(context, &mail).await?;
+        parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
         Ok(parser)
@@ -209,6 +224,8 @@ impl MimeMessage {
         } else if let Some(value) = self.get(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
+            } else if value == "ephemeral-timer-changed" {
+                self.is_system_message = SystemMessage::EphemeralTimerChanged;
             }
         }
         Ok(())
@@ -225,10 +242,24 @@ impl MimeMessage {
         }
     }
 
+    fn parse_videochat_headers(&mut self) {
+        if let Some(value) = self.get(HeaderDef::ChatContent).cloned() {
+            if value == "videochat-invitation" {
+                let instance = self.get(HeaderDef::ChatWebrtcRoom).cloned();
+                if let Some(part) = self.parts.first_mut() {
+                    part.typ = Viewtype::VideochatInvitation;
+                    part.param
+                        .set(Param::WebrtcRoom, instance.unwrap_or_default());
+                }
+            }
+        }
+    }
+
     /// Squashes mutlipart chat messages with attachment into single-part messages.
     ///
     /// Delta Chat sends attachments, such as images, in two-part messages, with the first message
     /// containing an explanation. If such a message is detected, first part can be safely dropped.
+    #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
             let need_drop = {
@@ -252,7 +283,8 @@ impl MimeMessage {
                 self.parts[0].msg = "".to_string();
 
                 // swap new with old
-                std::mem::replace(&mut self.parts[0], filepart);
+                self.parts.push(filepart); // push to the end
+                let _ = self.parts.swap_remove(0); // drops first element, replacing it with the last one in O(1)
             }
         }
     }
@@ -261,22 +293,21 @@ impl MimeMessage {
     fn parse_attachments(&mut self) {
         // Attachment messages should be squashed into a single part
         // before calling this function.
-        if self.parts.len() == 1 {
-            if self.parts[0].typ == Viewtype::Audio
-                && self.get(HeaderDef::ChatVoiceMessage).is_some()
-            {
-                let part_mut = &mut self.parts[0];
-                part_mut.typ = Viewtype::Voice;
+        if self.parts.len() != 1 {
+            return;
+        }
+
+        if let Some(mut part) = self.parts.pop() {
+            if part.typ == Viewtype::Audio && self.get(HeaderDef::ChatVoiceMessage).is_some() {
+                part.typ = Viewtype::Voice;
             }
-            if self.parts[0].typ == Viewtype::Image {
+            if part.typ == Viewtype::Image {
                 if let Some(value) = self.get(HeaderDef::ChatContent) {
                     if value == "sticker" {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.typ = Viewtype::Sticker;
+                        part.typ = Viewtype::Sticker;
                     }
                 }
             }
-            let part = &self.parts[0];
             if part.typ == Viewtype::Audio
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
@@ -284,17 +315,19 @@ impl MimeMessage {
                 if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.param.set_int(Param::Duration, duration_ms);
+                        part.param.set_int(Param::Duration, duration_ms);
                     }
                 }
             }
+
+            self.parts.push(part);
         }
     }
 
     fn parse_headers(&mut self, context: &Context) -> Result<()> {
         self.parse_system_message_headers(context)?;
         self.parse_avatar_headers();
+        self.parse_videochat_headers();
         self.squash_attachment_parts();
 
         if let Some(ref subject) = self.get_subject() {
@@ -310,12 +343,11 @@ impl MimeMessage {
                 }
             }
             if prepend_subject {
-                let subj = if let Some(n) = subject.find('[') {
-                    &subject[0..n]
-                } else {
-                    subject
-                }
-                .trim();
+                let subj = subject
+                    .find('[')
+                    .and_then(|n| subject.get(..n))
+                    .unwrap_or(subject)
+                    .trim();
 
                 if !subj.is_empty() {
                     for part in self.parts.iter_mut() {
@@ -352,7 +384,7 @@ impl MimeMessage {
         // just have send a message in the subject with an empty body.
         // Besides, we want to show something in case our incoming-processing
         // failed to properly handle an incoming message.
-        if self.parts.is_empty() && self.reports.is_empty() {
+        if self.parts.is_empty() && self.mdn_reports.is_empty() {
             let mut part = Part::default();
             part.typ = Viewtype::Text;
 
@@ -373,8 +405,7 @@ impl MimeMessage {
             Some(AvatarAction::Delete)
         } else {
             let mut i = 0;
-            while i != self.parts.len() {
-                let part = &mut self.parts[i];
+            while let Some(part) = self.parts.get_mut(i) {
                 if let Some(part_filename) = &part.org_filename {
                     if part_filename == &header_value {
                         if let Some(blob) = part.param.get(Param::File) {
@@ -391,6 +422,11 @@ impl MimeMessage {
         }
     }
 
+    /// Returns true if the message was encrypted as defined in
+    /// Autocrypt standard.
+    ///
+    /// This means the message was both encrypted and signed with a
+    /// valid signature.
     pub fn was_encrypted(&self) -> bool {
         !self.signatures.is_empty()
     }
@@ -526,6 +562,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
+                part.error = "Decryption failed".to_string();
 
                 self.parts.push(part);
 
@@ -537,7 +574,7 @@ impl MimeMessage {
                 contains exactly two body parts.  The first body
                 part is the body part over which the digital signature was created [...]
                 The second body part contains the control information necessary to
-                verify the digital signature." We simpliy take the first body part and
+                verify the digital signature." We simply take the first body part and
                 skip the rest.  (see
                 https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
                 for background information why we use encrypted+signed) */
@@ -548,10 +585,10 @@ impl MimeMessage {
             (mime::MULTIPART, "report") => {
                 /* RFC 6522: the first part is for humans, the second for machines */
                 if mail.subparts.len() >= 2 {
-                    if let Some(report_type) = mail.ctype.params.get("report-type") {
-                        if report_type == "disposition-notification" {
+                    match mail.ctype.params.get("report-type").map(|s| s as &str) {
+                        Some("disposition-notification") => {
                             if let Some(report) = self.process_report(context, mail)? {
-                                self.reports.push(report);
+                                self.mdn_reports.push(report);
                             }
 
                             // Add MDN part so we can track it, avoid
@@ -563,9 +600,21 @@ impl MimeMessage {
                             self.parts.push(part);
 
                             any_part_added = true;
-                        } else {
-                            /* eg. `report-type=delivery-status`;
-                            maybe we should show them as a little error icon */
+                        }
+                        // Some providers, e.g. Tiscali, forget to set the report-type. So, if it's None, assume that it might be delivery-status
+                        Some("delivery-status") | None => {
+                            if let Some(report) = self.process_delivery_status(context, mail)? {
+                                self.failure_report = Some(report);
+                            }
+
+                            // Add all parts (we need another part, preferrably text/plain, to show as an error message)
+                            for cur_data in mail.subparts.iter() {
+                                if self.parse_mime_recursive(context, cur_data).await? {
+                                    any_part_added = true;
+                                }
+                            }
+                        }
+                        Some(_) => {
                             if let Some(first) = mail.subparts.iter().next() {
                                 any_part_added = self.parse_mime_recursive(context, first).await?;
                             }
@@ -746,16 +795,11 @@ impl MimeMessage {
     }
 
     pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
-        if self.parts.is_empty() {
-            return;
+        if let Some(part) = self.parts.first_mut() {
+            part.typ = Viewtype::Text;
+            part.msg = format!("[{}]", error_msg.as_ref());
+            self.parts.truncate(1);
         }
-
-        let part = &mut self.parts[0];
-        part.typ = Viewtype::Text;
-        part.msg = format!("[{}]", error_msg.as_ref());
-        self.parts.truncate(1);
-
-        assert_eq!(self.parts.len(), 1);
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
@@ -806,7 +850,11 @@ impl MimeMessage {
         report: &mailparse::ParsedMail<'_>,
     ) -> Result<Option<Report>> {
         // parse as mailheaders
-        let report_body = report.subparts[1].get_body_raw()?;
+        let report_body = if let Some(subpart) = report.subparts.get(1) {
+            subpart.get_body_raw()?
+        } else {
+            bail!("Report does not have second MIME part");
+        };
         let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
         // must be present
@@ -838,24 +886,118 @@ impl MimeMessage {
         Ok(None)
     }
 
-    /// Handle reports (only MDNs for now)
-    pub async fn handle_reports(&self, context: &Context, from_id: u32, sent_timestamp: i64) {
-        if self.reports.is_empty() {
-            return;
+    fn process_delivery_status(
+        &self,
+        context: &Context,
+        report: &mailparse::ParsedMail<'_>,
+    ) -> Result<Option<FailureReport>> {
+        // parse as mailheaders
+        if let Some(original_msg) = report
+            .subparts
+            .iter()
+            .find(|p| p.ctype.mimetype.contains("rfc822") || p.ctype.mimetype == "message/global")
+        {
+            let report_body = original_msg.get_body_raw()?;
+            let (report_fields, _) = mailparse::parse_headers(&report_body)?;
+
+            if let Some(original_message_id) = report_fields
+                .get_header_value(HeaderDef::MessageId)
+                .and_then(|v| parse_message_id(&v).ok())
+            {
+                let mut to_list = get_all_addresses_from_header(&report.headers, |header_key| {
+                    header_key == "x-failed-recipients"
+                });
+                let to = if to_list.len() == 1 {
+                    Some(to_list.pop().unwrap())
+                } else {
+                    None // We do not know which recipient failed
+                };
+
+                return Ok(Some(FailureReport {
+                    rfc724_mid: original_message_id,
+                    failed_recipient: to.map(|s| s.addr),
+                }));
+            }
+
+            warn!(
+                context,
+                "ignoring unknown ndn-notification, Message-Id: {:?}",
+                report_fields.get_header_value(HeaderDef::MessageId)
+            );
         }
 
-        for report in &self.reports {
+        Ok(None)
+    }
+
+    /// Some providers like GMX and Yahoo do not send standard NDNs (Non Delivery notifications).
+    /// If you improve heuristics here you might also have to change prefetch_should_download() in imap/mod.rs.
+    /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
+    #[allow(clippy::indexing_slicing)]
+    async fn heuristically_parse_ndn(&mut self, context: &Context) -> Option<()> {
+        let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
+            let from = from.to_ascii_lowercase();
+            from.contains("mailer-daemon") || from.contains("mail-daemon")
+        } else {
+            false
+        };
+        if maybe_ndn && self.failure_report.is_none() {
+            lazy_static! {
+                static ref RE: regex::Regex = regex::Regex::new(r"Message-ID:(.*)").unwrap();
+            }
+            for captures in self
+                .parts
+                .iter()
+                .filter_map(|part| part.msg_raw.as_ref())
+                .flat_map(|part| part.lines())
+                .filter_map(|line| RE.captures(line))
+            {
+                if let Ok(original_message_id) = parse_message_id(&captures[1]) {
+                    if let Ok(Some(_)) =
+                        message::rfc724_mid_exists(context, &original_message_id).await
+                    {
+                        self.failure_report = Some(FailureReport {
+                            rfc724_mid: original_message_id,
+                            failed_recipient: None,
+                        })
+                    }
+                }
+            }
+        }
+        None // Always return None, we just return anything so that we can use the '?' operator.
+    }
+
+    /// Handle reports
+    /// (MDNs = Message Disposition Notification, the message was read
+    /// and NDNs = Non delivery notification, the message could not be delivered)
+    pub async fn handle_reports(
+        &self,
+        context: &Context,
+        from_id: u32,
+        sent_timestamp: i64,
+        parts: &[Part],
+    ) {
+        for report in &self.mdn_reports {
             for original_message_id in
                 std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
             {
                 if let Some((chat_id, msg_id)) =
-                    message::mdn_from_ext(context, from_id, original_message_id, sent_timestamp)
-                        .await
+                    message::handle_mdn(context, from_id, original_message_id, sent_timestamp).await
                 {
                     context.emit_event(Event::MsgRead { chat_id, msg_id });
                 }
             }
         }
+
+        if let Some(failure_report) = &self.failure_report {
+            let error = parts.iter().find(|p| p.typ == Viewtype::Text).map(|p| {
+                let msg = &p.msg;
+                msg.find("\n--- ")
+                    .and_then(|footer_start| msg.get(..footer_start))
+                    .unwrap_or(msg)
+                    .trim()
+            });
+            message::handle_ndn(context, failure_report, error).await
+        }
     }
 }
 
@@ -912,6 +1054,13 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
+#[derive(Debug)]
+pub(crate) struct FailureReport {
+    pub rfc724_mid: String,
+    pub failed_recipient: Option<String>,
+}
+
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
     // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
     let mut msgids = Vec::new();
@@ -955,6 +1104,7 @@ pub struct Part {
     pub bytes: usize,
     pub param: Params,
     org_filename: Option<String>,
+    pub error: String,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1117,7 +1267,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_mimeparser_crash() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1129,7 +1279,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_rfc724_mid_exists() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/mail_with_message_id.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1143,7 +1293,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_rfc724_mid_not_exists() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1201,7 +1351,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_parse_first_addr() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello@one.org, world@two.org\n\
                     Chat-Disposition-Notification-To: wrong\n\
                     Content-Type: text/plain\n\
@@ -1222,7 +1372,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_mimeparser_with_context() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello\n\
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
@@ -1272,7 +1422,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_mimeparser_with_avatars() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
@@ -1313,9 +1463,31 @@ mod tests {
         assert!(mimeparser.group_avatar.unwrap().is_change());
     }
 
+    #[async_std::test]
+    async fn test_mimeparser_with_videochat() {
+        let t = TestContext::new().await;
+
+        let raw = include_bytes!("../test-data/message/videochat_invitation.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        assert_eq!(mimeparser.parts.len(), 1);
+        assert_eq!(mimeparser.parts[0].typ, Viewtype::VideochatInvitation);
+        assert_eq!(
+            mimeparser.parts[0]
+                .param
+                .get(Param::WebrtcRoom)
+                .unwrap_or_default(),
+            "https://example.org/p2p/?roomname=6HiduoAn4xN"
+        );
+        assert!(mimeparser.parts[0]
+            .msg
+            .contains("https://example.org/p2p/?roomname=6HiduoAn4xN"));
+        assert_eq!(mimeparser.user_avatar, None);
+        assert_eq!(mimeparser.group_avatar, None);
+    }
+
     #[async_std::test]
     async fn test_mimeparser_message_kml() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Chat-Version: 1.0\n\
 From: foo <foo@example.org>\n\
 To: bar <bar@example.org>\n\
@@ -1360,7 +1532,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\
 
     #[async_std::test]
     async fn test_parse_mdn() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1401,7 +1573,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
+        assert_eq!(message.mdn_reports.len(), 1);
     }
 
     /// Test parsing multiple MDNs combined in a single message.
@@ -1410,7 +1582,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
     /// multipart MIME messages.
     #[async_std::test]
     async fn test_parse_multiple_mdns() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1481,12 +1653,12 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 2);
-        assert_eq!(message.reports.len(), 2);
+        assert_eq!(message.mdn_reports.len(), 2);
     }
 
     #[async_std::test]
     async fn test_parse_mdn_with_additional_message_ids() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1528,17 +1700,20 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
-        assert_eq!(message.reports[0].original_message_id, "foo@example.org");
+        assert_eq!(message.mdn_reports.len(), 1);
         assert_eq!(
-            &message.reports[0].additional_message_ids,
+            message.mdn_reports[0].original_message_id,
+            "foo@example.org"
+        );
+        assert_eq!(
+            &message.mdn_reports[0].additional_message_ids,
             &["foo@example.com", "foo@example.net"]
         );
     }
 
     #[async_std::test]
     async fn test_parse_inline_attachment() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
 From: sender@example.com
 To: receiver@example.com
@@ -1578,7 +1753,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 
     #[async_std::test]
     async fn parse_inline_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"Message-ID: <foobar@example.org>
 From: foo <foo@example.org>
 Subject: example
@@ -1624,7 +1799,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
     #[async_std::test]
     async fn parse_thunderbird_html_embedded_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"To: Alice <alice@example.org>
 From: Bob <bob@example.org>
 Subject: Test subject
@@ -1697,7 +1872,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
     // Outlook specifies filename in the "name" attribute of Content-Type
     #[async_std::test]
     async fn parse_outlook_html_embedded_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br##"From: Anonymous <anonymous@example.org>
 To: Anonymous <anonymous@example.org>
 Subject: Delta Chat is great stuff!

src/oauth2.rs

@@ -1,12 +1,16 @@
 //! OAuth 2 module
 
+use regex::Regex;
 use std::collections::HashMap;
 
+use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::provider;
+use crate::provider::Oauth2Authorizer;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
@@ -15,6 +19,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
+    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -24,8 +29,11 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
+    mx_pattern: None,
 };
 
+const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -33,6 +41,7 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
+    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -53,7 +62,7 @@ pub async fn dc_get_oauth2_url(
     addr: impl AsRef<str>,
     redirect_uri: impl AsRef<str>,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
             .set_raw_config(
@@ -81,7 +90,7 @@ pub async fn dc_get_oauth2_access_token(
     code: impl AsRef<str>,
     regenerate: bool,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -239,7 +248,7 @@ pub async fn dc_get_oauth2_addr(
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    let oauth2 = Oauth2::from_address(addr.as_ref()).await?;
     oauth2.get_userinfo?;
 
     if let Some(access_token) =
@@ -263,23 +272,56 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    fn from_address(addr: impl AsRef<str>) -> Option<Self> {
+    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
         let addr_normalized = normalize_addr(addr.as_ref());
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            match domain {
-                "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
-                | "narod.ru" => Some(OAUTH2_YANDEX),
-                _ => None,
+            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
+                match &provider.oauth2_authorizer {
+                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
+                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
+                    None => None, // provider known to not support oauth2, no mx-lookup required
+                }
+            } else {
+                Oauth2::lookup_mx(domain).await
             }
         } else {
             None
         }
     }
 
+    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
+        if let Ok(resolver) = resolver(
+            config::ResolverConfig::default(),
+            config::ResolverOpts::default(),
+        )
+        .await
+        {
+            for provider in OAUTH2_PROVIDERS.iter() {
+                if let Some(pattern) = provider.mx_pattern {
+                    let re = Regex::new(pattern).unwrap();
+
+                    let mut fqdn: String = String::from(domain.as_ref());
+                    if !fqdn.ends_with('.') {
+                        fqdn.push_str(".");
+                    }
+
+                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
+                        for rr in res.iter() {
+                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
+                                return Some(provider.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
         let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
@@ -362,25 +404,39 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_oauth_from_address() {
-        assert_eq!(Oauth2::from_address("hello@gmail.com"), Some(OAUTH2_GMAIL));
+    #[async_std::test]
+    async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com"),
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com"),
+            Oauth2::from_address("hello@googlemail.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.com").await,
+            Some(OAUTH2_YANDEX)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address("hello@yandex.ru"), Some(OAUTH2_YANDEX));
 
-        assert_eq!(Oauth2::from_address("hello@web.de"), None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
+    }
+
+    #[async_std::test]
+    async fn test_oauth_from_mx() {
+        assert_eq!(
+            Oauth2::from_address("hello@google.com").await,
+            Some(OAUTH2_GMAIL)
+        );
     }
 
     #[async_std::test]
     async fn test_dc_get_oauth2_addr() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_addr(&ctx.ctx, addr, code).await;
@@ -390,7 +446,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_url() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
         let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
@@ -400,7 +456,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_token() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false).await;

src/param.rs

@@ -66,10 +66,10 @@ pub enum Param {
     Arg4 = b'H',
 
     /// For Messages
-    Error = b'L',
+    AttachGroupImage = b'A',
 
     /// For Messages
-    AttachGroupImage = b'A',
+    WebrtcRoom = b'V',
 
     /// For Messages: space-separated list of messaged IDs of forwarded copies.
     ///
@@ -103,6 +103,9 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
+    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    LastSubject = b't',
+
     /// For Chats
     Devicetalk = b'D',
 
@@ -171,7 +174,7 @@ impl str::FromStr for Params {
             let key = key.unwrap_or_default().trim();
             let value = value.unwrap_or_default().trim();
 
-            if let Some(key) = Param::from_u8(key.as_bytes()[0]) {
+            if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
                 inner.insert(key, value.to_string());
             } else {
                 bail!("Unknown key: {}", key);
@@ -414,7 +417,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_params_file_fs_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
@@ -424,7 +427,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_params_file_blob() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
@@ -435,7 +438,7 @@ mod tests {
     // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
     #[async_std::test]
     async fn test_params_get_fileparam() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let fname = t.dir.path().join("foo");
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());

src/peerstate.rs

@@ -1,13 +1,12 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
 use std::collections::HashSet;
-use std::convert::TryFrom;
 use std::fmt;
 
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::context::Context;
-use crate::key::{DcKey, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
 
 #[derive(Debug)]
@@ -32,12 +31,12 @@ pub struct Peerstate<'a> {
     pub last_seen_autocrypt: i64,
     pub prefer_encrypt: EncryptPreference,
     pub public_key: Option<SignedPublicKey>,
-    pub public_key_fingerprint: Option<String>,
+    pub public_key_fingerprint: Option<Fingerprint>,
     pub gossip_key: Option<SignedPublicKey>,
     pub gossip_timestamp: i64,
-    pub gossip_key_fingerprint: Option<String>,
+    pub gossip_key_fingerprint: Option<Fingerprint>,
     pub verified_key: Option<SignedPublicKey>,
-    pub verified_key_fingerprint: Option<String>,
+    pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
     pub degrade_event: Option<DegradeEvent>,
 }
@@ -151,7 +150,7 @@ impl<'a> Peerstate<'a> {
     pub async fn from_fingerprint(
         context: &'a Context,
         _sql: &Sql,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
     ) -> Option<Peerstate<'a>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
@@ -160,13 +159,8 @@ impl<'a> Peerstate<'a> {
                      WHERE public_key_fingerprint=? COLLATE NOCASE \
                      OR gossip_key_fingerprint=? COLLATE NOCASE  \
                      ORDER BY public_key_fingerprint=? DESC;";
-
-        Self::from_stmt(
-            context,
-            query,
-            paramsv![fingerprint, fingerprint, fingerprint],
-        )
-        .await
+        let fp = fingerprint.hex();
+        Self::from_stmt(context, query, paramsv![fp, fp, fp]).await
     }
 
     async fn from_stmt(
@@ -189,33 +183,18 @@ impl<'a> Peerstate<'a> {
                 res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
                 res.gossip_timestamp = row.get(5)?;
 
-                res.public_key_fingerprint = row.get(7)?;
-                if res
-                    .public_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.public_key_fingerprint = None;
-                }
-                res.gossip_key_fingerprint = row.get(8)?;
-                if res
-                    .gossip_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.gossip_key_fingerprint = None;
-                }
-                res.verified_key_fingerprint = row.get(10)?;
-                if res
-                    .verified_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.verified_key_fingerprint = None;
-                }
+                res.public_key_fingerprint = row
+                    .get::<_, Option<String>>(7)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.gossip_key_fingerprint = row
+                    .get::<_, Option<String>>(8)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.verified_key_fingerprint = row
+                    .get::<_, Option<String>>(10)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
                 res.public_key = row
                     .get(4)
                     .ok()
@@ -238,7 +217,7 @@ impl<'a> Peerstate<'a> {
     pub fn recalc_fingerprint(&mut self) {
         if let Some(ref public_key) = self.public_key {
             let old_public_fingerprint = self.public_key_fingerprint.take();
-            self.public_key_fingerprint = Some(public_key.fingerprint().hex());
+            self.public_key_fingerprint = Some(public_key.fingerprint());
 
             if old_public_fingerprint.is_none()
                 || self.public_key_fingerprint.is_none()
@@ -253,7 +232,7 @@ impl<'a> Peerstate<'a> {
 
         if let Some(ref gossip_key) = self.gossip_key {
             let old_gossip_fingerprint = self.gossip_key_fingerprint.take();
-            self.gossip_key_fingerprint = Some(gossip_key.fingerprint().hex());
+            self.gossip_key_fingerprint = Some(gossip_key.fingerprint());
 
             if old_gossip_fingerprint.is_none()
                 || self.gossip_key_fingerprint.is_none()
@@ -344,11 +323,9 @@ impl<'a> Peerstate<'a> {
 
     pub fn render_gossip_header(&self, min_verified: PeerstateVerifiedStatus) -> Option<String> {
         if let Some(key) = self.peek_key(min_verified) {
-            // TODO: avoid cloning
-            let public_key = SignedPublicKey::try_from(key.clone()).ok()?;
             let header = Aheader::new(
                 self.addr.clone(),
-                public_key,
+                key.clone(), // TODO: avoid cloning
                 // Autocrypt 1.1.0 specification says that
                 // `prefer-encrypt` attribute SHOULD NOT be included,
                 // but we include it anyway to propagate encryption
@@ -387,7 +364,7 @@ impl<'a> Peerstate<'a> {
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
         verified: PeerstateVerifiedStatus,
     ) -> bool {
         if verified == PeerstateVerifiedStatus::BidirectVerified {
@@ -445,10 +422,10 @@ impl<'a> Peerstate<'a> {
                     self.public_key.as_ref().map(|k| k.to_bytes()),
                     self.gossip_timestamp,
                     self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    self.public_key_fingerprint,
-                    self.gossip_key_fingerprint,
+                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    self.verified_key_fingerprint,
+                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.addr,
                 ],
             ).await?;
@@ -469,15 +446,18 @@ impl<'a> Peerstate<'a> {
         Ok(())
     }
 
-    pub fn has_verified_key(&self, fingerprints: &HashSet<String>) -> bool {
-        if self.verified_key.is_some() && self.verified_key_fingerprint.is_some() {
-            let vkc = self.verified_key_fingerprint.as_ref().unwrap();
-            if fingerprints.contains(vkc) {
-                return true;
-            }
+    pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
+        if let Some(vkc) = &self.verified_key_fingerprint {
+            fingerprints.contains(vkc) && self.verified_key.is_some()
+        } else {
+            false
         }
+    }
+}
 
-        false
+impl From<crate::key::FingerprintError> for rusqlite::Error {
+    fn from(_source: crate::key::FingerprintError) -> Self {
+        Self::InvalidColumnType(0, "Invalid fingerprint".into(), rusqlite::types::Type::Text)
     }
 }
 
@@ -490,7 +470,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
         let pub_key = alice_keypair().public;
@@ -502,12 +482,12 @@ mod tests {
             last_seen_autocrypt: 11,
             prefer_encrypt: EncryptPreference::Mutual,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
             gossip_key: Some(pub_key.clone()),
             gossip_timestamp: 12,
-            gossip_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            gossip_key_fingerprint: Some(pub_key.fingerprint()),
             verified_key: Some(pub_key.clone()),
-            verified_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            verified_key_fingerprint: Some(pub_key.fingerprint()),
             to_save: Some(ToSave::All),
             degrade_event: None,
         };
@@ -525,7 +505,7 @@ mod tests {
         peerstate.to_save = None;
         assert_eq!(peerstate, peerstate_new);
         let peerstate_new2 =
-            Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint().hex())
+            Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint())
                 .await
                 .expect("failed to load peerstate from db");
         assert_eq!(peerstate, peerstate_new2);
@@ -533,7 +513,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_double_create() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
         let pub_key = alice_keypair().public;
 
@@ -544,7 +524,7 @@ mod tests {
             last_seen_autocrypt: 11,
             prefer_encrypt: EncryptPreference::Mutual,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
             gossip_key: None,
             gossip_timestamp: 12,
             gossip_key_fingerprint: None,
@@ -566,7 +546,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_with_empty_gossip_key_save_to_db() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
         let pub_key = alice_keypair().public;
@@ -578,7 +558,7 @@ mod tests {
             last_seen_autocrypt: 11,
             prefer_encrypt: EncryptPreference::Mutual,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
             gossip_key: None,
             gossip_timestamp: 12,
             gossip_key_fingerprint: None,

src/pgp.rs

@@ -18,7 +18,7 @@ use rand::{thread_rng, CryptoRng, Rng};
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::key::DcKey;
+use crate::key::{DcKey, Fingerprint};
 use crate::keyring::Keyring;
 
 pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
@@ -272,12 +272,20 @@ pub async fn pk_encrypt(
     .await
 }
 
+/// Decrypts the message with keys from the private key keyring.
+///
+/// Receiver private keys are provided in
+/// `private_keys_for_decryption`.
+///
+/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// of all keys from the `public_keys_for_validation` keyring that
+/// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
 pub async fn pk_decrypt(
     ctext: Vec<u8>,
     private_keys_for_decryption: Keyring<SignedSecretKey>,
     public_keys_for_validation: Keyring<SignedPublicKey>,
-    ret_signature_fingerprints: Option<&mut HashSet<String>>,
+    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
 ) -> Result<Vec<u8>> {
     let msgs = async_std::task::spawn_blocking(move || {
         let cursor = Cursor::new(ctext);
@@ -290,36 +298,41 @@ pub async fn pk_decrypt(
     })
     .await?;
 
-    ensure!(!msgs.is_empty(), "No valid messages found");
+    if let Some(msg) = msgs.into_iter().next() {
+        // get_content() will decompress the message if needed,
+        // but this avoids decompressing it again to check signatures
+        let msg = msg.decompress()?;
 
-    let content = match msgs[0].get_content()? {
-        Some(content) => content,
-        None => bail!("Decrypted message is empty"),
-    };
+        let content = match msg.get_content()? {
+            Some(content) => content,
+            None => bail!("The decrypted message is empty"),
+        };
 
-    if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-        if !public_keys_for_validation.is_empty() {
-            let fingerprints = async_std::task::spawn_blocking(move || {
-                let dec_msg = &msgs[0];
+        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
+            if !public_keys_for_validation.is_empty() {
+                let fingerprints = async_std::task::spawn_blocking(move || {
+                    let pkeys = public_keys_for_validation.keys();
 
-                let pkeys = public_keys_for_validation.keys();
-
-                let mut fingerprints = Vec::new();
-                for pkey in pkeys {
-                    if dec_msg.verify(&pkey.primary_key).is_ok() {
-                        let fp = DcKey::fingerprint(pkey).hex();
-                        fingerprints.push(fp);
+                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                        for pkey in pkeys {
+                            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                                let fp = DcKey::fingerprint(pkey);
+                                fingerprints.push(fp);
+                            }
+                        }
                     }
-                }
-                fingerprints
-            })
-            .await;
+                    fingerprints
+                })
+                .await;
 
-            ret_signature_fingerprints.extend(fingerprints);
+                ret_signature_fingerprints.extend(fingerprints);
+            }
         }
+        Ok(content)
+    } else {
+        bail!("No valid messages found");
     }
-
-    Ok(content)
 }
 
 /// Symmetric encryption.
@@ -352,11 +365,13 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
         let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
         let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-        ensure!(!msgs.is_empty(), "No valid messages found");
-
-        match msgs[0].get_content()? {
-            Some(content) => Ok(content),
-            None => bail!("Decrypted message is empty"),
+        if let Some(msg) = msgs.first() {
+            match msg.get_content()? {
+                Some(content) => Ok(content),
+                None => bail!("Decrypted message is empty"),
+            }
+        } else {
+            bail!("No valid messages found")
         }
     })
     .await
@@ -474,7 +489,7 @@ mod tests {
         decrypt_keyring.add(KEYS.alice_secret.clone());
         let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
@@ -492,7 +507,7 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
@@ -511,7 +526,7 @@ mod tests {
         let mut keyring = Keyring::new();
         keyring.add(KEYS.alice_secret.clone());
         let empty_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             keyring,
@@ -531,7 +546,7 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.bob_public.clone());
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
@@ -549,7 +564,7 @@ mod tests {
         let mut decrypt_keyring = Keyring::new();
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let sig_check_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_UNSIGNED.as_bytes().to_vec(),
             decrypt_keyring,

src/provider/data.rs

@@ -19,6 +19,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // aol.md: aol.com
@@ -30,6 +32,23 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // arcor.de.md: arcor.de
+    static ref P_ARCOR_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/arcor-de",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.arcor.de", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.arcor.de", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // autistici.org.md: autistici.org
@@ -43,6 +62,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -56,16 +77,81 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // chello.at.md: chello.at
+    static ref P_CHELLO_AT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/chello-at",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "mail.mymagenta.at", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.mymagenta.at", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_COMCAST: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/comcast",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // dismail.de.md: dismail.de
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISMAIL_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dismail-de",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // disroot.md: disroot.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISROOT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/disroot",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
+
+    // dubby.org.md: dubby.org
+    static ref P_DUBBY_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dubby-org",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // example.com.md: example.com, example.org
     static ref P_EXAMPLE_COM: Provider = Provider {
@@ -78,6 +164,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // fastmail.md: fastmail.com
@@ -89,6 +177,26 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // five.chat.md: five.chat
+    static ref P_FIVE_CHAT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/five-chat",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // freenet.de.md: freenet.de
@@ -102,6 +210,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -115,6 +225,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -129,10 +241,35 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // hey.com.md: hey.com
+    static ref P_HEY_COM: Provider = Provider {
+        status: Status::BROKEN,
+        before_login_hint: "hey.com does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to hey.com.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/hey-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // i.ua.md: i.ua
-    // - skipping provider with status OK and no special things to do
+    static ref P_I_UA: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/i-ua",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // icloud.md: icloud.com, me.com, mac.com
     static ref P_ICLOUD: Provider = Provider {
@@ -145,19 +282,61 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // kolst.com.md: kolst.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KOLST_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kolst-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // kontent.com.md: kontent.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KONTENT_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kontent-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAIL_RU: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mail-ru",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAILBOX_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mailbox-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // nauta.cu.md: nauta.cu
     static ref P_NAUTA_CU: Provider = Provider {
@@ -178,6 +357,8 @@ lazy_static::lazy_static! {
             ConfigDefault { key: Config::E2eeEnabled, value: "0" },
             ConfigDefault { key: Config::MediaQuality, value: "1" },
         ]),
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -191,9 +372,11 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // posteo.md: posteo.de
+    // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
     static ref P_POSTEO: Provider = Provider {
         status: Status::OK,
         before_login_hint: "",
@@ -204,6 +387,8 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -215,16 +400,48 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // riseup.net.md: riseup.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_RISEUP_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/riseup-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // rogers.com.md: rogers.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_ROGERS_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/rogers-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // systemli.org.md: systemli.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_SYSTEMLI_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/systemli-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
@@ -235,6 +452,8 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // testrun.md: testrun.org
@@ -248,7 +467,14 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // tiscali.it.md: tiscali.it
@@ -262,13 +488,35 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // ukr.net.md: ukr.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_UKR_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/ukr-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // vfemail.md: vfemail.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_VFEMAIL: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/vfemail",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
     static ref P_WEB_DE: Provider = Provider {
@@ -282,12 +530,14 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
     static ref P_YAHOO: Provider = Provider {
         status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with your Yahoo email address you have to allow \"less secure apps\" in the Yahoo webinterface.",
+        before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yahoo",
         server: vec![
@@ -295,9 +545,11 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // yandex.ru.md: yandex.ru, yandex.com
+    // yandex.ru.md: yandex.com, yandex.by, yandex.kz, yandex.ru, yandex.ua, ya.ru, narod.ru
     static ref P_YANDEX_RU: Provider = Provider {
         status: Status::PREPARATION,
         before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
@@ -306,6 +558,8 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -319,16 +573,26 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
+        ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("chello.at", &*P_CHELLO_AT),
+        ("xfinity.com", &*P_COMCAST),
+        ("comcast.net", &*P_COMCAST),
+        ("dismail.de", &*P_DISMAIL_DE),
+        ("disroot.org", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -341,9 +605,19 @@ lazy_static::lazy_static! {
         ("gmx.info", &*P_GMX_NET),
         ("gmx.biz", &*P_GMX_NET),
         ("gmx.com", &*P_GMX_NET),
+        ("hey.com", &*P_HEY_COM),
+        ("i.ua", &*P_I_UA),
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
+        ("kolst.com", &*P_KOLST_COM),
+        ("kontent.com", &*P_KONTENT_COM),
+        ("mail.ru", &*P_MAIL_RU),
+        ("inbox.ru", &*P_MAIL_RU),
+        ("bk.ru", &*P_MAIL_RU),
+        ("list.ru", &*P_MAIL_RU),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
@@ -351,12 +625,65 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
+        ("posteo.af", &*P_POSTEO),
+        ("posteo.at", &*P_POSTEO),
+        ("posteo.be", &*P_POSTEO),
+        ("posteo.ch", &*P_POSTEO),
+        ("posteo.cl", &*P_POSTEO),
+        ("posteo.co", &*P_POSTEO),
+        ("posteo.co.uk", &*P_POSTEO),
+        ("posteo.com.br", &*P_POSTEO),
+        ("posteo.cr", &*P_POSTEO),
+        ("posteo.cz", &*P_POSTEO),
+        ("posteo.dk", &*P_POSTEO),
+        ("posteo.ee", &*P_POSTEO),
+        ("posteo.es", &*P_POSTEO),
+        ("posteo.eu", &*P_POSTEO),
+        ("posteo.fi", &*P_POSTEO),
+        ("posteo.gl", &*P_POSTEO),
+        ("posteo.gr", &*P_POSTEO),
+        ("posteo.hn", &*P_POSTEO),
+        ("posteo.hr", &*P_POSTEO),
+        ("posteo.hu", &*P_POSTEO),
+        ("posteo.ie", &*P_POSTEO),
+        ("posteo.in", &*P_POSTEO),
+        ("posteo.is", &*P_POSTEO),
+        ("posteo.jp", &*P_POSTEO),
+        ("posteo.la", &*P_POSTEO),
+        ("posteo.li", &*P_POSTEO),
+        ("posteo.lt", &*P_POSTEO),
+        ("posteo.lu", &*P_POSTEO),
+        ("posteo.me", &*P_POSTEO),
+        ("posteo.mx", &*P_POSTEO),
+        ("posteo.my", &*P_POSTEO),
+        ("posteo.net", &*P_POSTEO),
+        ("posteo.nl", &*P_POSTEO),
+        ("posteo.no", &*P_POSTEO),
+        ("posteo.nz", &*P_POSTEO),
+        ("posteo.org", &*P_POSTEO),
+        ("posteo.pe", &*P_POSTEO),
+        ("posteo.pl", &*P_POSTEO),
+        ("posteo.pm", &*P_POSTEO),
+        ("posteo.pt", &*P_POSTEO),
+        ("posteo.ro", &*P_POSTEO),
+        ("posteo.ru", &*P_POSTEO),
+        ("posteo.se", &*P_POSTEO),
+        ("posteo.sg", &*P_POSTEO),
+        ("posteo.si", &*P_POSTEO),
+        ("posteo.tn", &*P_POSTEO),
+        ("posteo.uk", &*P_POSTEO),
+        ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("rogers.com", &*P_ROGERS_COM),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
+        ("ukr.net", &*P_UKR_NET),
+        ("vfemail.net", &*P_VFEMAIL),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),
         ("flirt.ms", &*P_WEB_DE),
@@ -399,8 +726,13 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
-        ("yandex.ru", &*P_YANDEX_RU),
         ("yandex.com", &*P_YANDEX_RU),
+        ("yandex.by", &*P_YANDEX_RU),
+        ("yandex.kz", &*P_YANDEX_RU),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("yandex.ua", &*P_YANDEX_RU),
+        ("ya.ru", &*P_YANDEX_RU),
+        ("narod.ru", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/provider/mod.rs

@@ -35,6 +35,13 @@ pub enum UsernamePattern {
     EMAILLOCALPART = 2,
 }
 
+#[derive(Debug, PartialEq)]
+#[repr(u8)]
+pub enum Oauth2Authorizer {
+    Yandex = 1,
+    Gmail = 2,
+}
+
 #[derive(Debug)]
 pub struct Server {
     pub protocol: Protocol,
@@ -72,6 +79,8 @@ pub struct Provider {
     pub overview_page: &'static str,
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
+    pub strict_tls: bool,
+    pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
 impl Provider {

src/provider/update.py

@@ -100,6 +100,12 @@ def process_data(data, file):
 
     config_defaults = process_config_defaults(data)
 
+    strict_tls = data.get("strict_tls", False)
+    strict_tls = "true" if strict_tls else "false"
+
+    oauth2 = data.get("oauth2", "")
+    oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -111,6 +117,8 @@ def process_data(data, file):
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
         provider += "        config_defaults: " + config_defaults + ",\n"
+        provider += "        strict_tls: " + strict_tls + ",\n"
+        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -121,11 +129,11 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
-        out_all += "    // - skipping provider with status OK and no special things to do\n\n"
-    else:
-        out_all += provider
-        out_domains += domains
+
+    # also add provider with no special things to do -
+    # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
+    out_all += provider
+    out_domains += domains
 
 
 def process_file(file):

src/qr.rs

@@ -10,8 +10,7 @@ use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::key::dc_format_fingerprint;
-use crate::key::dc_normalize_fingerprint;
+use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
@@ -69,6 +68,7 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
 /// scheme: `OPENPGP4FPR:FINGERPRINT#a=ADDR&n=NAME&i=INVITENUMBER&s=AUTH`
 ///     or: `OPENPGP4FPR:FINGERPRINT#a=ADDR&g=GROUPNAME&x=GROUPID&i=INVITENUMBER&s=AUTH`
+#[allow(clippy::indexing_slicing)]
 async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
@@ -80,6 +80,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         Some(pair) => pair,
         None => (payload, ""),
     };
+    let fingerprint: Fingerprint = match fingerprint.parse() {
+        Ok(fp) => fp,
+        Err(err) => {
+            return Error::new(err)
+                .context("Failed to parse fingerprint in QR code")
+                .into()
+        }
+    };
 
     // replace & with \n to match expected param format
     let fragment = fragment.replace('&', "\n");
@@ -128,13 +136,6 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         None
     };
 
-    let fingerprint = dc_normalize_fingerprint(fingerprint);
-
-    // ensure valid fingerprint
-    if fingerprint.len() != 40 {
-        return format_err!("Bad fingerprint length in QR code").into();
-    }
-
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
@@ -161,7 +162,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr)).await;
         } else {
             lot.state = LotState::QrFprWithoutAddr;
-            lot.text1 = Some(dc_format_fingerprint(&fingerprint));
+            lot.text1 = Some(fingerprint.to_string());
         }
     } else if let Some(addr) = addr {
         if grpid.is_some() && grpname.is_some() {
@@ -187,13 +188,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 }
 
 /// scheme: `DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3`
+#[allow(clippy::indexing_slicing)]
 fn decode_account(_context: &Context, qr: &str) -> Lot {
     let payload = &qr[DCACCOUNT_SCHEME.len()..];
 
     let mut lot = Lot::new();
 
     if let Ok(url) = url::Url::parse(payload) {
-        if url.scheme() == "https" {
+        if url.scheme() == "http" || url.scheme() == "https" {
             lot.state = LotState::QrAccount;
             lot.text1 = url.host_str().map(|x| x.to_string());
         } else {
@@ -217,6 +219,7 @@ struct CreateAccountResponse {
 /// take a qr of the type DC_QR_ACCOUNT, parse it's parameters,
 /// download additional information from the contained url and set the parameters.
 /// on success, a configure::configure() should be able to log in to the account
+#[allow(clippy::indexing_slicing)]
 pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
 
@@ -240,6 +243,7 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error
 /// Extract address for the mailto scheme.
 ///
 /// Scheme: `mailto:addr...?subject=...&body=..`
+#[allow(clippy::indexing_slicing)]
 async fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let payload = &qr[MAILTO_SCHEME.len()..];
 
@@ -261,6 +265,7 @@ async fn decode_mailto(context: &Context, qr: &str) -> Lot {
 /// Extract address for the smtp scheme.
 ///
 /// Scheme: `SMTP:addr...:subject...:body...`
+#[allow(clippy::indexing_slicing)]
 async fn decode_smtp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[SMTP_SCHEME.len()..];
 
@@ -283,6 +288,7 @@ async fn decode_smtp(context: &Context, qr: &str) -> Lot {
 /// Scheme: `MATMSG:TO:addr...;SUB:subject...;BODY:body...;`
 ///
 /// There may or may not be linebreaks after the fields.
+#[allow(clippy::indexing_slicing)]
 async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     // Does not work when the text `TO:` is used in subject/body _and_ TO: is not the first field.
     // we ignore this case.
@@ -316,14 +322,15 @@ lazy_static! {
 /// Extract address for the matmsg scheme.
 ///
 /// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
+#[allow(clippy::indexing_slicing)]
 async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
         .captures(qr)
-        .map(|caps| {
-            let last_name = &caps[1];
-            let first_name = &caps[2];
+        .and_then(|caps| {
+            let last_name = caps.get(1)?.as_str().trim();
+            let first_name = caps.get(2)?.as_str().trim();
 
-            format!("{} {}", first_name.trim(), last_name.trim())
+            Some(format!("{} {}", first_name, last_name))
         })
         .unwrap_or_default();
 
@@ -383,11 +390,11 @@ fn normalize_address(addr: &str) -> Result<String, Error> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_decode_http() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "http://www.hello.com").await;
 
@@ -399,7 +406,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_https() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "https://www.hello.com").await;
 
@@ -411,7 +418,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_text() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "I am so cool").await;
 
@@ -423,7 +430,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_vcard() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -441,7 +448,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_matmsg() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -459,7 +466,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_mailto() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -485,7 +492,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_smtp() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld").await;
 
@@ -499,7 +506,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_group() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -528,7 +535,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_secure_join() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -556,7 +563,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_without_addr() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -591,7 +598,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_account() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -613,10 +620,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_account_bad_scheme() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
-            "DCACCOUNT:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "DCACCOUNT:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
@@ -625,7 +632,7 @@ mod tests {
         // Test it again with lowercased "dcaccount:" uri scheme
         let res = check_qr(
             &ctx.ctx,
-            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "dcaccount:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);

src/scheduler.rs

@@ -5,7 +5,7 @@ use async_std::task;
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::smtp::Smtp;
+use crate::{config::Config, message::MsgId, smtp::Smtp};
 
 pub(crate) struct StopToken;
 
@@ -32,36 +32,12 @@ impl Context {
         self.scheduler.read().await.maybe_network().await;
     }
 
-    pub(crate) async fn interrupt_inbox(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_inbox(probe_network)
-            .await;
+    pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_inbox(info).await;
     }
 
-    pub(crate) async fn interrupt_sentbox(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_sentbox(probe_network)
-            .await;
-    }
-
-    pub(crate) async fn interrupt_mvbox(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_mvbox(probe_network)
-            .await;
-    }
-
-    pub(crate) async fn interrupt_smtp(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_smtp(probe_network)
-            .await;
+    pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_smtp(info).await;
     }
 }
 
@@ -79,20 +55,16 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         // track number of continously executed jobs
         let mut jobs_loaded = 0;
-        let mut probe_network = false;
+        let mut info = InterruptInfo::default();
         loop {
-            match job::load_next(&ctx, Thread::Imap, probe_network).await {
+            match job::load_next(&ctx, Thread::Imap, &info).await {
                 Some(job) if jobs_loaded <= 20 => {
                     jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
-                    probe_network = false;
+                    info = Default::default();
                 }
                 Some(job) => {
                     // Let the fetch run, but return back to the job afterwards.
@@ -102,7 +74,18 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 }
                 None => {
                     jobs_loaded = 0;
-                    probe_network = fetch_idle(&ctx, &mut connection).await;
+
+                    // Expunge folder if needed, e.g. if some jobs have
+                    // deleted messages on the server.
+                    if let Err(err) = connection.maybe_close_folder(&ctx).await {
+                        warn!(ctx, "failed to close folder: {:?}", err);
+                    }
+
+                    info = if ctx.get_config_bool(Config::InboxWatch).await {
+                        fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
+                    } else {
+                        connection.fake_idle(&ctx, None).await
+                    };
                 }
             }
         }
@@ -119,15 +102,18 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 }
 
 async fn fetch(ctx: &Context, connection: &mut Imap) {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error_network!(ctx, "{}", err);
+                return;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
         }
         None => {
             warn!(ctx, "Can not fetch inbox folder, not set");
@@ -136,16 +122,20 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     }
 }
 
-async fn fetch_idle(ctx: &Context, connection: &mut Imap) -> bool {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
+    match ctx.get_config(folder).await {
         Some(watch_folder) => {
+            // connect and fake idle if unable to connect
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                warn!(ctx, "imap connection failed: {}", err);
+                return connection.fake_idle(&ctx, None).await;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
 
             // idle
             if connection.can_idle() {
@@ -153,15 +143,16 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap) -> bool {
                     .idle(&ctx, Some(watch_folder))
                     .await
                     .unwrap_or_else(|err| {
-                        error!(ctx, "{}", err);
-                        false
+                        connection.trigger_reconnect();
+                        warn!(ctx, "{}", err);
+                        InterruptInfo::new(false, None)
                     })
             } else {
                 connection.fake_idle(&ctx, Some(watch_folder)).await
             }
         }
         None => {
-            warn!(ctx, "Can not watch inbox folder, not set");
+            warn!(ctx, "Can not watch {} folder, not set", folder);
             connection.fake_idle(&ctx, None).await
         }
     }
@@ -171,7 +162,7 @@ async fn simple_imap_loop(
     ctx: Context,
     started: Sender<()>,
     inbox_handlers: ImapConnectionHandlers,
-    folder: impl AsRef<str>,
+    folder: Config,
 ) {
     use futures::future::FutureExt;
 
@@ -187,44 +178,9 @@ async fn simple_imap_loop(
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         loop {
-            match get_watch_folder(&ctx, folder.as_ref()).await {
-                Some(watch_folder) => {
-                    // fetch
-                    connection
-                        .fetch(&ctx, &watch_folder)
-                        .await
-                        .unwrap_or_else(|err| {
-                            error!(ctx, "{}", err);
-                        });
-
-                    // idle
-                    if connection.can_idle() {
-                        connection
-                            .idle(&ctx, Some(watch_folder))
-                            .await
-                            .unwrap_or_else(|err| {
-                                error!(ctx, "{}", err);
-                                false
-                            });
-                    } else {
-                        connection.fake_idle(&ctx, Some(watch_folder)).await;
-                    }
-                }
-                None => {
-                    warn!(
-                        &ctx,
-                        "No watch folder found for {}, skipping",
-                        folder.as_ref()
-                    );
-                    connection.fake_idle(&ctx, None).await;
-                }
-            }
+            fetch_idle(&ctx, &mut connection, folder).await;
         }
     };
 
@@ -254,18 +210,18 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         started.send(()).await;
         let ctx = ctx1;
 
-        let mut probe_network = false;
+        let mut interrupt_info = Default::default();
         loop {
-            match job::load_next(&ctx, Thread::Smtp, probe_network).await {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
                 Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
-                    probe_network = false;
+                    interrupt_info = Default::default();
                 }
                 None => {
                     // Fake Idle
                     info!(ctx, "smtp fake idle - started");
-                    probe_network = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
                     info!(ctx, "smtp fake idle - interrupted")
                 }
             }
@@ -290,61 +246,64 @@ impl Scheduler {
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
         let (inbox, inbox_handlers) = ImapConnectionState::new();
 
+        let (inbox_start_send, inbox_start_recv) = channel(1);
+        let (mvbox_start_send, mvbox_start_recv) = channel(1);
+        let mut mvbox_handle = None;
+        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let mut sentbox_handle = None;
+        let (smtp_start_send, smtp_start_recv) = channel(1);
+
+        let ctx1 = ctx.clone();
+        let inbox_handle = Some(task::spawn(async move {
+            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
+        }));
+
+        if ctx.get_config_bool(Config::MvboxWatch).await {
+            let ctx1 = ctx.clone();
+            mvbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    mvbox_start_send,
+                    mvbox_handlers,
+                    Config::ConfiguredMvboxFolder,
+                )
+                .await
+            }));
+        } else {
+            mvbox_start_send.send(()).await;
+        }
+
+        if ctx.get_config_bool(Config::SentboxWatch).await {
+            let ctx1 = ctx.clone();
+            sentbox_handle = Some(task::spawn(async move {
+                simple_imap_loop(
+                    ctx1,
+                    sentbox_start_send,
+                    sentbox_handlers,
+                    Config::ConfiguredSentboxFolder,
+                )
+                .await
+            }));
+        } else {
+            sentbox_start_send.send(()).await;
+        }
+
+        let ctx1 = ctx.clone();
+        let smtp_handle = Some(task::spawn(async move {
+            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
+        }));
+
         *self = Scheduler::Running {
             inbox,
             mvbox,
             sentbox,
             smtp,
-            inbox_handle: None,
-            mvbox_handle: None,
-            sentbox_handle: None,
-            smtp_handle: None,
+            inbox_handle,
+            mvbox_handle,
+            sentbox_handle,
+            smtp_handle,
         };
 
-        let (inbox_start_send, inbox_start_recv) = channel(1);
-        if let Scheduler::Running { inbox_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *inbox_handle = Some(task::spawn(async move {
-                inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
-            }));
-        }
-
-        let (mvbox_start_send, mvbox_start_recv) = channel(1);
-        if let Scheduler::Running { mvbox_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *mvbox_handle = Some(task::spawn(async move {
-                simple_imap_loop(
-                    ctx1,
-                    mvbox_start_send,
-                    mvbox_handlers,
-                    "configured_mvbox_folder",
-                )
-                .await
-            }));
-        }
-
-        let (sentbox_start_send, sentbox_start_recv) = channel(1);
-        if let Scheduler::Running { sentbox_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *sentbox_handle = Some(task::spawn(async move {
-                simple_imap_loop(
-                    ctx1,
-                    sentbox_start_send,
-                    sentbox_handlers,
-                    "configured_sentbox_folder",
-                )
-                .await
-            }));
-        }
-
-        let (smtp_start_send, smtp_start_recv) = channel(1);
-        if let Scheduler::Running { smtp_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *smtp_handle = Some(task::spawn(async move {
-                smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
-            }));
-        }
-
         // wait for all loops to be started
         if let Err(err) = inbox_start_recv
             .recv()
@@ -364,34 +323,34 @@ impl Scheduler {
             return;
         }
 
-        self.interrupt_inbox(true)
-            .join(self.interrupt_mvbox(true))
-            .join(self.interrupt_sentbox(true))
-            .join(self.interrupt_smtp(true))
+        self.interrupt_inbox(InterruptInfo::new(true, None))
+            .join(self.interrupt_mvbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_smtp(InterruptInfo::new(true, None)))
             .await;
     }
 
-    async fn interrupt_inbox(&self, probe_network: bool) {
+    async fn interrupt_inbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref inbox, .. } = self {
-            inbox.interrupt(probe_network).await;
+            inbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_mvbox(&self, probe_network: bool) {
+    async fn interrupt_mvbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref mvbox, .. } = self {
-            mvbox.interrupt(probe_network).await;
+            mvbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_sentbox(&self, probe_network: bool) {
+    async fn interrupt_sentbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref sentbox, .. } = self {
-            sentbox.interrupt(probe_network).await;
+            sentbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_smtp(&self, probe_network: bool) {
+    async fn interrupt_smtp(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref smtp, .. } = self {
-            smtp.interrupt(probe_network).await;
+            smtp.interrupt(info).await;
         }
     }
 
@@ -433,10 +392,18 @@ impl Scheduler {
                 smtp_handle,
                 ..
             } => {
-                inbox_handle.take().expect("inbox not started").await;
-                mvbox_handle.take().expect("mvbox not started").await;
-                sentbox_handle.take().expect("sentbox not started").await;
-                smtp_handle.take().expect("smtp not started").await;
+                if let Some(handle) = inbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = mvbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = sentbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = smtp_handle.take() {
+                    handle.await;
+                }
 
                 *self = Scheduler::Stopped;
             }
@@ -460,7 +427,7 @@ struct ConnectionState {
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
-    idle_interrupt_sender: Sender<bool>,
+    idle_interrupt_sender: Sender<InterruptInfo>,
 }
 
 impl ConnectionState {
@@ -472,9 +439,9 @@ impl ConnectionState {
         self.shutdown_receiver.recv().await.ok();
     }
 
-    async fn interrupt(&self, probe_network: bool) {
+    async fn interrupt(&self, info: InterruptInfo) {
         // Use try_send to avoid blocking on interrupts.
-        self.idle_interrupt_sender.try_send(probe_network).ok();
+        self.idle_interrupt_sender.try_send(info).ok();
     }
 }
 
@@ -508,8 +475,8 @@ impl SmtpConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self, probe_network: bool) {
-        self.state.interrupt(probe_network).await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -518,12 +485,11 @@ impl SmtpConnectionState {
     }
 }
 
-#[derive(Debug)]
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
     shutdown_sender: Sender<()>,
-    idle_interrupt_receiver: Receiver<bool>,
+    idle_interrupt_receiver: Receiver<InterruptInfo>,
 }
 
 #[derive(Debug)]
@@ -556,8 +522,8 @@ impl ImapConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self, probe_network: bool) {
-        self.state.interrupt(probe_network).await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -573,20 +539,17 @@ struct ImapConnectionHandlers {
     shutdown_sender: Sender<()>,
 }
 
-async fn get_watch_folder(context: &Context, config_name: impl AsRef<str>) -> Option<String> {
-    match context
-        .sql
-        .get_raw_config(context, config_name.as_ref())
-        .await
-    {
-        Some(name) => Some(name),
-        None => {
-            if config_name.as_ref() == "configured_inbox_folder" {
-                // initialized with old version, so has not set configured_inbox_folder
-                Some("INBOX".to_string())
-            } else {
-                None
-            }
+#[derive(Default, Debug)]
+pub struct InterruptInfo {
+    pub probe_network: bool,
+    pub msg_id: Option<MsgId>,
+}
+
+impl InterruptInfo {
+    pub fn new(probe_network: bool, msg_id: Option<MsgId>) -> Self {
+        Self {
+            probe_network,
+            msg_id,
         }
     }
 }

src/securejoin.rs

@@ -14,7 +14,7 @@ use crate::e2ee::*;
 use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, DcKey, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -73,8 +73,6 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     ====   Step 1 in "Setup verified contact" protocol   ====
     =======================================================*/
 
-    let fingerprint: String;
-
     ensure_secret_key_exists(context).await.ok();
 
     // invitenumber will be used to allow starting the handshake,
@@ -95,7 +93,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         .await
         .unwrap_or_default();
 
-    fingerprint = match get_self_fingerprint(context).await {
+    let fingerprint: Fingerprint = match get_self_fingerprint(context).await {
         Some(fp) => fp,
         None => {
             return None;
@@ -116,7 +114,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
 
             Some(format!(
                 "OPENPGP4FPR:{}#a={}&g={}&x={}&i={}&s={}",
-                fingerprint,
+                fingerprint.hex(),
                 self_addr_urlencoded,
                 &group_name_urlencoded,
                 &chat.grpid,
@@ -131,7 +129,11 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         // parameters used: a=n=i=s=
         Some(format!(
             "OPENPGP4FPR:{}#a={}&n={}&i={}&s={}",
-            fingerprint, self_addr_urlencoded, self_name_urlencoded, &invitenumber, &auth,
+            fingerprint.hex(),
+            self_addr_urlencoded,
+            self_name_urlencoded,
+            &invitenumber,
+            &auth,
         ))
     };
 
@@ -140,9 +142,9 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     qr
 }
 
-async fn get_self_fingerprint(context: &Context) -> Option<String> {
+async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
     match SignedPublicKey::load_self(context).await {
-        Ok(key) => Some(key.fingerprint().hex()),
+        Ok(key) => Some(key.fingerprint()),
         Err(_) => {
             warn!(context, "get_self_fingerprint(): failed to load key");
             None
@@ -249,7 +251,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
             chat_id_2_contact_id(context, contact_chat_id).await,
             400
         );
-        let own_fingerprint = get_self_fingerprint(context).await.unwrap_or_default();
+        let own_fingerprint = get_self_fingerprint(context).await;
 
         // Bob -> Alice
         if let Err(err) = send_handshake_msg(
@@ -261,7 +263,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
                 "vc-request-with-auth"
             },
             get_qr_attr!(context, auth).to_string(),
-            Some(own_fingerprint),
+            own_fingerprint,
             if join_vg {
                 get_qr_attr!(context, text2).to_string()
             } else {
@@ -311,7 +313,7 @@ async fn send_handshake_msg(
     contact_chat_id: ChatId,
     step: &str,
     param2: impl AsRef<str>,
-    fingerprint: Option<String>,
+    fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
 ) -> Result<(), HandshakeError> {
     let mut msg = Message::default();
@@ -328,7 +330,7 @@ async fn send_handshake_msg(
         msg.param.set(Param::Arg2, param2);
     }
     if let Some(fp) = fingerprint {
-        msg.param.set(Param::Arg3, fp);
+        msg.param.set(Param::Arg3, fp.hex());
     }
     if !grpid.as_ref().is_empty() {
         msg.param.set(Param::Arg4, grpid.as_ref());
@@ -350,9 +352,8 @@ async fn send_handshake_msg(
 }
 
 async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32 {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
-    if contacts.len() == 1 {
-        contacts[0]
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        contact_id
     } else {
         0
     }
@@ -360,17 +361,14 @@ async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32
 
 async fn fingerprint_equals_sender(
     context: &Context,
-    fingerprint: impl AsRef<str>,
+    fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
 ) -> bool {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
-
-    if contacts.len() == 1 {
-        if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
             if let Some(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
-                let fingerprint_normalized = dc_normalize_fingerprint(fingerprint.as_ref());
                 if peerstate.public_key_fingerprint.is_some()
-                    && &fingerprint_normalized == peerstate.public_key_fingerprint.as_ref().unwrap()
+                    && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
                     return true;
                 }
@@ -397,6 +395,8 @@ pub(crate) enum HandshakeError {
     NoSelfAddr,
     #[error("Failed to send message")]
     MsgSendFailed(#[source] Error),
+    #[error("Failed to parse fingerprint")]
+    BadFingerprint(#[from] crate::key::FingerprintError),
 }
 
 /// What to do with a Secure-Join handshake message after it was handled.
@@ -423,6 +423,7 @@ pub(crate) enum HandshakeMessage {
 /// When handle_securejoin_handshake() is called,
 /// the message is not yet filed in the database;
 /// this is done by receive_imf() later on as needed.
+#[allow(clippy::indexing_slicing)]
 pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
     mime_message: &MimeMessage,
@@ -516,10 +517,11 @@ pub(crate) async fn handle_securejoin_handshake(
                 // no error, just aborted somehow or a mail from another handshake
                 return Ok(HandshakeMessage::Ignore);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
             let auth = get_qr_attr!(context, auth).to_string();
 
-            if !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice) {
+            if !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -576,8 +578,9 @@ pub(crate) async fn handle_securejoin_handshake(
             ==========================================================*/
 
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -588,7 +591,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !encrypted_and_signed(context, mime_message, &fingerprint) {
+            if !encrypted_and_signed(context, mime_message, Some(&fingerprint)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -625,7 +628,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -673,7 +676,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     contact_chat_id,
                     "vc-contact-confirm",
                     "",
-                    Some(fingerprint.clone()),
+                    Some(fingerprint),
                     "",
                 )
                 .await?;
@@ -709,7 +712,8 @@ pub(crate) async fn handle_securejoin_handshake(
                 );
                 return Ok(abort_retval);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
@@ -731,7 +735,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 true
             };
             if vg_expect_encrypted
-                && !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice)
+                && !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice))
             {
                 could_not_establish_secure_connection(
                     context,
@@ -888,7 +892,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
             if !encrypted_and_signed(
                 context,
                 mime_message,
-                get_self_fingerprint(context).await.unwrap_or_default(),
+                get_self_fingerprint(context).await.as_ref(),
             ) {
                 could_not_establish_secure_connection(
                     context,
@@ -898,8 +902,9 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -910,7 +915,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -967,16 +972,13 @@ async fn could_not_establish_secure_connection(
     error!(context, "{} ({})", &msg, details);
 }
 
-async fn mark_peer_as_verified(
-    context: &Context,
-    fingerprint: impl AsRef<str>,
-) -> Result<(), Error> {
+async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
     if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint.as_ref()).await
+        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await
     {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
-            fingerprint.as_ref(),
+            fingerprint,
             PeerstateVerifiedStatus::BidirectVerified,
         ) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
@@ -990,7 +992,7 @@ async fn mark_peer_as_verified(
     }
     bail!(
         "could not mark peer as verified for fingerprint {}",
-        fingerprint.as_ref()
+        fingerprint.hex()
     );
 }
 
@@ -1001,29 +1003,24 @@ async fn mark_peer_as_verified(
 fn encrypted_and_signed(
     context: &Context,
     mimeparser: &MimeMessage,
-    expected_fingerprint: impl AsRef<str>,
+    expected_fingerprint: Option<&Fingerprint>,
 ) -> bool {
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
         false
-    } else if mimeparser.signatures.is_empty() {
-        warn!(context, "Message not signed.",);
-        false
-    } else if expected_fingerprint.as_ref().is_empty() {
-        warn!(context, "Fingerprint for comparison missing.",);
-        false
-    } else if !mimeparser
-        .signatures
-        .contains(expected_fingerprint.as_ref())
-    {
-        warn!(
-            context,
-            "Message does not match expected fingerprint {}.",
-            expected_fingerprint.as_ref(),
-        );
-        false
+    } else if let Some(expected_fingerprint) = expected_fingerprint {
+        if !mimeparser.signatures.contains(expected_fingerprint) {
+            warn!(
+                context,
+                "Message does not match expected fingerprint {}.", expected_fingerprint,
+            );
+            false
+        } else {
+            true
+        }
     } else {
-        true
+        warn!(context, "Fingerprint for comparison missing.");
+        false
     }
 }
 

src/simplify.rs

@@ -7,14 +7,15 @@
 // but for non-delta-compatibility, that seems to be better.
 // (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
 pub fn escape_message_footer_marks(text: &str) -> String {
-    if text.starts_with("--") {
-        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
+    if let Some(text) = text.strip_prefix("--") {
+        "-\u{200B}-".to_string() + &text.replace("\n--", "\n-\u{200B}-")
     } else {
         text.replace("\n--", "\n-\u{200B}-")
     }
 }
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
+#[allow(clippy::indexing_slicing)]
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
     let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
@@ -41,6 +42,7 @@ fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
 
 /// Remove nonstandard footer and a boolean indicating whether such
 /// footer was removed.
+#[allow(clippy::indexing_slicing)]
 fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     for (ix, &line) in lines.iter().enumerate() {
         if line == "--"
@@ -107,6 +109,7 @@ fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     }
 }
 
+#[allow(clippy::indexing_slicing)]
 fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     let mut last_quoted_line = None;
     for (l, line) in lines.iter().enumerate().rev() {
@@ -132,6 +135,7 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     }
 }
 
+#[allow(clippy::indexing_slicing)]
 fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     let mut last_quoted_line = None;
     let mut has_quoted_headline = false;

src/smtp/mod.rs

@@ -2,7 +2,7 @@
 
 pub mod send;
 
-use std::time::{Duration, Instant};
+use std::time::{Duration, SystemTime};
 
 use async_smtp::smtp::client::net::*;
 use async_smtp::*;
@@ -10,8 +10,9 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
-use crate::login_param::{dc_build_tls, LoginParam};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::oauth2::*;
+use crate::provider::get_provider_info;
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -44,9 +45,8 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Default, DebugStub)]
-pub struct Smtp {
-    #[debug_stub(some = "SmtpTransport")]
+#[derive(Default)]
+pub(crate) struct Smtp {
     transport: Option<smtp::SmtpTransport>,
 
     /// Email address we are sending from.
@@ -55,7 +55,7 @@ pub struct Smtp {
     /// Timestamp of last successful send/receive network interaction
     /// (eg connect or send succeeded). On initialization and disconnect
     /// it is set to None.
-    last_success: Option<Instant>,
+    last_success: Option<SystemTime>,
 }
 
 impl Smtp {
@@ -76,7 +76,11 @@ impl Smtp {
     /// have been successfully used the last 60 seconds
     pub async fn has_maybe_stale_connection(&self) -> bool {
         if let Some(last_success) = self.last_success {
-            Instant::now().duration_since(last_success).as_secs() > 60
+            SystemTime::now()
+                .duration_since(last_success)
+                .unwrap_or_default()
+                .as_secs()
+                > 60
         } else {
             false
         }
@@ -113,7 +117,14 @@ impl Smtp {
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls_config = dc_build_tls(lp.smtp_certificate_checks);
+        let provider = get_provider_info(&lp.addr);
+        let strict_tls = match lp.smtp_certificate_checks {
+            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Strict => true,
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => false,
+        };
+        let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
@@ -181,7 +192,7 @@ impl Smtp {
         }
 
         self.transport = Some(trans);
-        self.last_success = Some(Instant::now());
+        self.last_success = Some(SystemTime::now());
 
         context.emit_event(Event::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",

src/smtp/send.rs

@@ -53,7 +53,7 @@ impl Smtp {
                 "Message len={} was smtp-sent to {}",
                 message_len, recipients_display
             )));
-            self.last_success = Some(std::time::Instant::now());
+            self.last_success = Some(std::time::SystemTime::now());
 
             Ok(())
         } else {

src/sql.rs

@@ -13,6 +13,7 @@ use crate::chat::{update_device_icon, update_saved_messages_icon};
 use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::start_ephemeral_timers;
 use crate::param::*;
 use crate::peerstate::*;
 
@@ -49,7 +50,7 @@ pub enum Error {
 pub type Result<T> = std::result::Result<T, Error>;
 
 /// A wrapper around the underlying Sqlite3 object.
-#[derive(DebugStub)]
+#[derive(Debug)]
 pub struct Sql {
     pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
 }
@@ -568,16 +569,24 @@ pub async fn housekeeping(context: &Context) {
         }
     }
 
+    if let Err(err) = start_ephemeral_timers(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot start ephemeral timers: {}", err
+        );
+    }
+
     if let Err(err) = prune_tombstones(context).await {
         warn!(
             context,
-            "Houskeeping: Cannot prune message tombstones: {}", err
+            "Housekeeping: Cannot prune message tombstones: {}", err
         );
     }
 
     info!(context, "Housekeeping done.",);
 }
 
+#[allow(clippy::indexing_slicing)]
 fn is_file_in_use(files_in_use: &HashSet<String>, namespc_opt: Option<&str>, name: &str) -> bool {
     let name_to_check = if let Some(namespc) = namespc_opt {
         let name_len = name.len();
@@ -593,11 +602,9 @@ fn is_file_in_use(files_in_use: &HashSet<String>, namespc_opt: Option<&str>, nam
 }
 
 fn maybe_add_file(files_in_use: &mut HashSet<String>, file: impl AsRef<str>) {
-    if !file.as_ref().starts_with("$BLOBDIR") {
-        return;
+    if let Some(file) = file.as_ref().strip_prefix("$BLOBDIR/") {
+        files_in_use.insert(file.to_string());
     }
-
-    files_in_use.insert(file.as_ref()[9..].into());
 }
 
 async fn maybe_add_from_param(
@@ -1241,6 +1248,41 @@ async fn open(
                 .await?;
             sql.set_raw_config_int(context, "dbversion", 63).await?;
         }
+        if dbversion < 64 {
+            info!(context, "[migration] v64");
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN error TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 64).await?;
+        }
+        if dbversion < 65 {
+            info!(context, "[migration] v65");
+            sql.execute(
+                "ALTER TABLE chats ADD COLUMN ephemeral_timer INTEGER",
+                paramsv![],
+            )
+            .await?;
+            // Timer value in seconds. For incoming messages this
+            // timer starts when message is read, so we want to have
+            // the value stored here until the timer starts.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            // Timestamp indicating when the message should be
+            // deleted. It is convenient to store it here because UI
+            // needs this value to display how much time is left until
+            // the message is deleted.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 65).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1303,10 +1345,12 @@ mod test {
         maybe_add_file(&mut files, "$BLOBDIR/hello");
         maybe_add_file(&mut files, "$BLOBDIR/world.txt");
         maybe_add_file(&mut files, "world2.txt");
+        maybe_add_file(&mut files, "$BLOBDIR");
 
         assert!(files.contains("hello"));
         assert!(files.contains("world.txt"));
         assert!(!files.contains("world2.txt"));
+        assert!(!files.contains("$BLOBDIR"));
     }
 
     #[test]

src/stock.rs

@@ -130,7 +130,9 @@ pub enum StockMessage {
     ))]
     AcSetupMsgBody = 43,
 
-    #[strum(props(fallback = "Cannot login as %1$s."))]
+    #[strum(props(
+        fallback = "Cannot login as \"%1$s\". Please check if the email address and the password are correct."
+    ))]
     CannotLogin = 60,
 
     #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
@@ -177,8 +179,43 @@ pub enum StockMessage {
                     however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
     WelcomeMessage = 71,
 
-    #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
+    #[strum(props(fallback = "Unknown sender for this chat. See 'info' for more details."))]
     UnknownSenderForChat = 72,
+
+    #[strum(props(fallback = "Message from %1$s"))]
+    SubjectForNewContact = 73,
+
+    #[strum(props(fallback = "Failed to send message to %1$s."))]
+    FailedSendingTo = 74,
+
+    #[strum(props(fallback = "Message deletion timer is disabled."))]
+    MsgEphemeralTimerDisabled = 75,
+
+    // A fallback message for unknown timer values.
+    // "s" stands for "second" SI unit here.
+    #[strum(props(fallback = "Message deletion timer is set to %1$s s."))]
+    MsgEphemeralTimerEnabled = 76,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 minute."))]
+    MsgEphemeralTimerMinute = 77,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 hour."))]
+    MsgEphemeralTimerHour = 78,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 day."))]
+    MsgEphemeralTimerDay = 79,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 week."))]
+    MsgEphemeralTimerWeek = 80,
+
+    #[strum(props(fallback = "Message deletion timer is set to 4 weeks."))]
+    MsgEphemeralTimerFourWeeks = 81,
+
+    #[strum(props(fallback = "Video chat invitation"))]
+    VideochatInvitation = 82,
+
+    #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
+    VideochatInviteMsgBody = 83,
 }
 
 /*
@@ -328,10 +365,10 @@ impl Context {
         let action1 = action.trim_end_matches('.');
         match from_id {
             0 => action,
-            1 => {
+            DC_CONTACT_ID_SELF => {
                 self.stock_string_repl_str(StockMessage::MsgActionByMe, action1)
                     .await
-            } // DC_CONTACT_ID_SELF
+            }
             _ => {
                 let displayname = Contact::get_by_id(self, from_id)
                     .await
@@ -403,7 +440,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_stock_translation() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::NoMessages, "xyz".to_string())
             .await
@@ -413,7 +450,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_stock_translation_wrong_replacements() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert!(t
             .ctx
             .set_stock_translation(StockMessage::NoMessages, "xyz %1$s ".to_string())
@@ -428,7 +465,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_str() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx.stock_str(StockMessage::NoMessages).await,
             "No messages."
@@ -437,7 +474,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_str() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         // uses %1$s substitution
         assert_eq!(
             t.ctx
@@ -450,7 +487,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_int() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_string_repl_int(StockMessage::MsgAddMember, 42)
@@ -461,7 +498,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_str2() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar")
@@ -472,7 +509,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_simple() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
@@ -483,7 +520,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_me() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(
@@ -499,7 +536,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_me_with_displayname() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         Contact::create(&t.ctx, "Alice", "alice@example.com")
             .await
             .expect("failed to create contact");
@@ -518,7 +555,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_other_with_displayname() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let contact_id = {
             Contact::create(&t.ctx, "Alice", "alice@example.com")
                 .await
@@ -542,7 +579,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_grp_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(
@@ -558,7 +595,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_grp_name_other() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let id = Contact::create(&t.ctx, "Alice", "alice@example.com")
             .await
             .expect("failed to create contact");
@@ -573,7 +610,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_update_device_chats() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.ok();
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 2);

src/test_utils.rs

@@ -18,26 +18,59 @@ pub(crate) struct TestContext {
     pub dir: TempDir,
 }
 
-/// Create a new, opened [TestContext] using given callback.
-///
-/// The [Context] will be opened with the SQLite database named
-/// "db.sqlite" in the [TestContext.dir] directory.
-///
-/// [Context]: crate::context::Context
-pub(crate) async fn test_context() -> TestContext {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let ctx = Context::new("FakeOs".into(), dbfile.into()).await.unwrap();
-    TestContext { ctx, dir }
-}
+impl TestContext {
+    /// Create a new [TestContext].
+    ///
+    /// The [Context] will be created and have an SQLite database named "db.sqlite" in the
+    /// [TestContext.dir] directory.  This directory is cleaned up when the [TestContext] is
+    /// dropped.
+    ///
+    /// [Context]: crate::context::Context
+    pub async fn new() -> Self {
+        let dir = tempdir().unwrap();
+        let dbfile = dir.path().join("db.sqlite");
+        let ctx = Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Self { ctx, dir }
+    }
 
-/// Return a dummy [TestContext].
-///
-/// The context will be opened and use the SQLite database as
-/// specified in [test_context] but there is no callback hooked up,
-/// i.e. [Context::call_cb] will always return `0`.
-pub(crate) async fn dummy_context() -> TestContext {
-    test_context().await
+    /// Create a new configured [TestContext].
+    ///
+    /// This is a shortcut which automatically calls [TestContext::configure_alice] after
+    /// creating the context.
+    pub async fn new_alice() -> Self {
+        let t = Self::new().await;
+        t.configure_alice().await;
+        t
+    }
+
+    /// Configure with alice@example.com.
+    ///
+    /// The context will be fake-configured as the alice user, with a pre-generated secret
+    /// key.  The email address of the user is returned as a string.
+    pub async fn configure_alice(&self) -> String {
+        let keypair = alice_keypair();
+        self.configure_addr(&keypair.addr.to_string()).await;
+        key::store_self_keypair(&self.ctx, &keypair, key::KeyPairUse::Default)
+            .await
+            .expect("Failed to save Alice's key");
+        keypair.addr.to_string()
+    }
+
+    /// Configure as a given email address.
+    ///
+    /// The context will be configured but the key will not be pre-generated so if a key is
+    /// used the fingerprint will be different every time.
+    pub async fn configure_addr(&self, addr: &str) {
+        self.ctx.set_config(Config::Addr, Some(addr)).await.unwrap();
+        self.ctx
+            .set_config(Config::ConfiguredAddr, Some(addr))
+            .await
+            .unwrap();
+        self.ctx
+            .set_config(Config::Configured, Some("1"))
+            .await
+            .unwrap();
+    }
 }
 
 /// Load a pre-generated keypair for alice@example.com from disk.
@@ -60,20 +93,6 @@ pub(crate) fn alice_keypair() -> key::KeyPair {
     }
 }
 
-/// Creates Alice with a pre-generated keypair.
-///
-/// Returns the address of the keypair created (alice@example.com).
-pub(crate) async fn configure_alice_keypair(ctx: &Context) -> String {
-    let keypair = alice_keypair();
-    ctx.set_config(Config::ConfiguredAddr, Some(&keypair.addr.to_string()))
-        .await
-        .unwrap();
-    key::store_self_keypair(&ctx, &keypair, key::KeyPairUse::Default)
-        .await
-        .expect("Failed to save Alice's key");
-    keypair.addr.to_string()
-}
-
 /// Load a pre-generated keypair for bob@example.net from disk.
 ///
 /// Like [alice_keypair] but a different key and identity.

test-data/message/gmail_ndn.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a1c:b4d7:0:0:0:0:0 with SMTP id d206csp3026053wmf;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Received: by 2002:a5d:4651:: with SMTP id j17mr19532177wrs.50.1589819005555;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1589819005; cv=none;
+        d=google.com; s=arc-20160816;
+        b=IZbNnzzuYzTFuqfuZwpd3ehqpYYGpn31c8DsfGbQ8rpbS0OTTROkVYvihQl8Ne/8X/
+         brEWsrcmaCh9WpFMzpI+cp/TY39uusnI6qdp5rcgrFmFgoANtwf3TBBj1+f7wBPn46BP
+         dQOUsg/J8KVfvzVgvL1x4uyJ0m9QirDgJeJ/BvrswbTleRQK7oY3fIireUCDxj6r2lCB
+         1Z0TKw1mgIb1LiFMZz8kvCNn3R4KSFnwS8rIju0hYwnsioNiExVQgumXL+RVkEZ9BMzf
+         UdoWIAw3VW+MOZFTpfLCEfgIPtLg/gtE0Q1P+a3KKpi8dkPiV2n6DGMecy9lTLtdhCXt
+         pnaA==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=nNP0DktrSjdBaFfhhoDi2O9KVKM0iXE5ZgubQ0q0ff68Z6Ke7c8dDBXEsZoToI0s4Y
+         w90KyJFpgMJLFmP3iVDRqCfohi2y1HGdWg5VXQPTvzM7+YozZRlbNNV9UsuyRY91CXrJ
+         a2XREBgB+LPMGQivwcHtUMZfyNv/4uiwWivk+92ySNDhxqOiDt4R5Jak/7RkZMFwQpsE
+         JGwk6asM6VqZlihkF24lKv3pPaob6feyX3wD5N0+Mqiy1kQTj2JkpQk6nkTmdf0gapZe
+         fOhU1NkbNfbuS3U7m2gEUiyktE+MhV/MgAzgBhm9bgNt2gQLVWju8rHkPndfv1PDmEkC
+         FsYQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id s18sor5584435wrb.25.2020.05.18.09.23.25
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=dPisws+OwGFyOy0a612XYZgvz5T71GcJRJtU068/Tce8vN/+ggIQtUsZnZtsphe71v
+         2NvfP9ULxR4cXvomTvhrYAk19KdxN/S7SeyBbmXv3x/tg+DBVCmmPS/6RXrcl6Ms3Hkw
+         uPFQ9S3KcvHe/2bcb5LSTA/stIP4tuxxAXvsX2j+MjPYPWKAl50jkSbWK98U0Q0U+MTl
+         pKaaC9s9iEBafac8BFZCy4DfpumKlemNEyRa3cSV2hw+DYHKA5peModrK1A2tcsfstFF
+         rZi8yF/D90RIFbE04DI2QCxB3trsChNF1aYF06aSzI//wsfM1+lb+uGPi0YVkw3n4HrX
+         Xw4w==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=A/NCOtgbpA7VzB1G7ZFo8TA2FfrjuqjGdwMrJr3yXe21FrBFwzssprJwOkynqoVLkK
+         iJU7uMF/KTcQPDEmOLFThzFfe5GCx7eJtZPhwY+FbBlC5sq4I55/xaQLd0gOZ1BYXwMn
+         2bk169d2aoukbaLbGSQZF3d9atd+/e48YzkRxpmUoLcrWk2LcHAeQIG7SgT9pfX5DKPr
+         VpxM5/GMVEBbTRhBIWCeVSfpYCs80l0xEeTC3/B5lzpzMVDE8QCW6Dwh75b4Tb2K6yru
+         Zsy5ZpRmwv0wrkrb2vM+pl4IMkaF7s8XosIvlIT++fQV5xDFItT4atpykZvSDB92RKV0
+         8lEA==
+X-Gm-Message-State: AOAM532RG/PT3ChZHBCDORGLtAjKvX8TGBuOy+AxrnEaJT6v1ieb+VV1
+	+ejly+/6UthxHYlkOJYAszCSgL4dKVFotoVaN7LhEA==
+X-Google-Smtp-Source: ABdhPJz6veVKWhomCL4gK+whrybuMzHCDCq8AowgQvi7sobpMoM/k9CDw79jo1j3OUcTz6MEeUYLxEXuNIuu4zyoS7kVtsUYryGFHAI=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545185wrv.159.1589819005394;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000012d63005a5ee9520"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a5d:5183:: with SMTP id k3mr13704211wrv.159; Mon, 18 May
+ 2020 09:23:25 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ec2b67d.1c69fb81.213af.67a5.GMR@mx.google.com>
+Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+In-Reply-To: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000012d63005a5ee9520
+Content-Type: multipart/related; boundary="00000000000012dc0005a5ee952f"
+
+--00000000000012dc0005a5ee952f
+Content-Type: multipart/alternative; boundary="00000000000012dc0705a5ee9530"
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--00000000000012dc0705a5ee9530--
+--00000000000012dc0005a5ee952f
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--00000000000012dc0005a5ee952f--
+--00000000000012d63005a5ee9520
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Original-Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+Last-Attempt-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+
+--00000000000012d63005a5ee9520
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=a185ogBcMzF9whNVWvuyUoUunNZk3Vc1kEIFmPkX0IxLpAFcI+fOQajOSromGl7Oyi
+         yecLwQevpww2Xd0XjZ3UkZvrI9m9koRmh0QeoHvgTRORiVwj08+PVc3N4F9bCO4w9i0J
+         ir7SSsJqBCDovoIFSFDyNa64vs6Nxno0cH/DaPG7pVTdD+3jfB7nLXIsMQYeX+1eP6rB
+         UhKxH82r7Mh9CI2PWDQpVtGj63AMUEyHgE9Ou08PWbbKjrQOasoG3Tw8tB1GoN1XYssM
+         rxOTgWEoTiduZ35AUH6h+eChOn9OHuI3SPECcVob70Qndayia3dMKfHMO6sEx9J0Wpic
+         29vg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=miGIfL5BgnkD3wQvS34RtGwRRoh+8gJT5sFFfdX/hVyG/dvjXfdwP4yyNWr8ox8iY2
+         BLlahS4y4VGcbG1e2aYjurnWNytGu6utQcZax/uUngJ0bTOwXW1VaIiEZtqd6gTV+8d/
+         rrfQ459+4vXqIoQf0+Oi/U6dWwgJvPPjjRiToWdF3vIJE8R1iTRdZbW4lkgxSADbmskg
+         noT/gWGWblHtR6uuGuKGJ3bkhJKCBnjavKh0LlbWEeFBZfmVNPRvzEFWHjBDdu5wvSL5
+         0QJ+Qn0Orfn5CJuN3xPfzT1S2rI2iYZx37KX9zyMnZEx0ilkTYqCtBPWkrXRYDSXcxYS
+         Y1ag==
+X-Gm-Message-State: AOAM531vhwpXiK8M12286dOJx0Q5fBl9ZaH6BJKts93GoxvPv0xdryP0
+	jg9wYmoP5MUHudsxAMCYDFsCUMVx2PEywyIsaQqklw==
+X-Google-Smtp-Source: ABdhPJxlVJtTODM3pZZSTbbpAAAQRU8XbmuosDF9fgQZmVwxGZSzRWl22o+moppVRU/r8xMAyf0r3+qXwEBe1vZfjZo=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545162wrv.159.1589819005034;
+ Mon, 18 May 2020 09:23:25 -0700 (PDT)
+MIME-Version: 1.0
+From: <alice@gmail.com>
+Date: Mon, 18 May 2020 18:23:39 +0200
+Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+Subject: Kommt sowieso nicht an
+To: assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000000d652a05a5ee95df"
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/plain; charset="UTF-8"
+
+Wollte nur was testen
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">Wollte nur was testen<br></div>
+
+--0000000000000d652a05a5ee95df--
+
+--00000000000012d63005a5ee9520--

test-data/message/gmail_ndn_group.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a02:6629:0:0:0:0:0 with SMTP id k41csp368502jac;
+        Wed, 10 Jun 2020 05:17:57 -0700 (PDT)
+X-Received: by 2002:a6b:1448:: with SMTP id 69mr2898530iou.83.1591791475733;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1591791475; cv=none;
+        d=google.com; s=arc-20160816;
+        b=a0vSKJPbMtGYFnuk1ye/gnnV00Zvva4OOJTMOyfm13xMJD0YAhzGVfa7Z+wn5sQ8dw
+         VAxpmDHCkjp4jol0C1iutiq2Nl0qma819oFPuuoMLLatKQXHpo8Jt+sL3MnwNR7J5bZC
+         1c6Fjk75EIsRWhJd1HCkm44A6UYHxqqsTnzQCaNiHbjsRsvbggxwlMGSrZ4silxqSDvo
+         Pzd/YDLCvsnZNSNIjIckKAwtGmY6sXctZ+JnOTykXAyL32Milfwy1vRL9xm10Q14biTR
+         +qaIQp4E6WE63g1WHvfAjs0Dru7DalUh4GGl/NAwqVhY1gVyRD5E9/nODyHAfxjvaxDD
+         4sMw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=OJbgbrktMKyczw25z/ib7lSdRX80PEK3Myh9fj4q6mDlXmPPv//Gv069znRQ4QbadM
+         HUXZH0WLMZcGyqI6SvGL/noxQ1O8yP0FYJJKTkoX0Gk2hHzfaE3x1scOP/o2FMMQXIFm
+         S4CgGBD6HHzBJYj/rSL3gzqLzx1Id/z5kTeDvH2cn8JJAcCE2q/nhjTyWUb87geoNlDJ
+         A1HRrLHK/0JOyRjHfg2zZCqIvSi1xmpiHStMyL9mfVyrQs98tsPxaOkJHjLplFARoPlr
+         mmmDvsFg7MPvFqkkANzz4JDHidnfKRULCgnrVj1yTU66UagUpQEGjZqz8/99YuU6nt1t
+         81sQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id w14sor16686480iow.23.2020.06.10.05.17.55
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=aO4aNy7CUOk9O4Jnsue/DvMFY6Ph0C34AbpoxJH+mLZpOmt/KYGCGYWgunZgamF15U
+         Vm8JY5yLKGwkTz2m3abDnKNP4fpl6zeZ5fyk5LvXH2Jema0iocHai6pJZBoFGPnonNmd
+         MscTf1sEltbOxwfOmM1BRHX34c1jW0+8Yd2+Nhg2DPvzuq1brOVin6bUV4VX5EeeuNqT
+         ZTewjJVPmO/B5NQhdpG81FO5w4hKSQ/VzZXnap2thMf3gOmnaoR+tbsnOIAiklcLdJ7b
+         57SKUwI041pwSmh9dffs0STl2GvMRSJyGCtBqMnzXgflqoGTcnPflWgR3LXHM/MIA0q8
+         WqRQ==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=iORAzNvXegQ8oSp4RYb/S168muAiBox769seMk49kDBIvXwI+N8P4mUZq/zDi8DmQd
+         +wlLzVzowQq6EofiSpjOJWT9IC/k8otk15PMGtgHE4BGSSeKn7L30d3ocQS93HzYnLmA
+         VBlHBdFTKrsfKhe2+CQyCosTDGRpbkQLuRRyhxChEq8ltvaOHgbu1+eCeb9PsPuh6OxH
+         kvTHJZeA9A+eLOl26pBmqGIWkr7FlYW0wI6YPoEs9WXX5LSFOQs6fm/9l366eIR7IFFI
+         ihX5LrZl/Cf0lwwYX7fqIMgnHy1K+QnKuEb+dRQGqLbxdIEls9bXIF98iPQVkEWzgSZy
+         ip8Q==
+X-Gm-Message-State: AOAM531ahfHE6oS9/nuni8pNf9bwC+DXAcaLV0owBwNCj9kcTPLCCNhX
+	W1JNciK0ivEIVB4dgiyLE/5K7iKbEznQhqyG9Bi1QA==
+X-Google-Smtp-Source: ABdhPJygljUXswH0ycJyHmXVthi5IjlDvP8QdYlMdHUPKEtgIZeUk69Acti5LnswGhg63T9/L0PuGZGBM5XE5BsP0mMNNDRZyt+DgnE=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990163jab.84.1591791475516;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000074432a05a7b9d512"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a05:6638:101c:: with SMTP id r28mr3059870jab.84; Wed, 10 Jun
+ 2020 05:17:55 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ee0cf73.1c69fb81.6888.c2f4.GMR@mx.google.com>
+Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+In-Reply-To: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000074432a05a7b9d512
+Content-Type: multipart/related; boundary="000000000000745e0805a7b9d51b"
+
+--000000000000745e0805a7b9d51b
+Content-Type: multipart/alternative; boundary="000000000000745e1705a7b9d51c"
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--000000000000745e1705a7b9d51c--
+--000000000000745e0805a7b9d51b
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--000000000000745e0805a7b9d51b--
+--00000000000074432a05a7b9d512
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+X-Original-Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+Last-Attempt-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+
+--00000000000074432a05a7b9d512
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=eyr60XbgOrgHoZFpRYzw9WQIR7aEBaYKWhiEcqdnugB+hn0W2KVcTkKiL2C6zSF+jh
+         l+lM+dNZZTUcMqWx4kVgTVtqwUNea8OUqe+WLqx04ULwdKZn1okbKYovaiavCLKOKDnf
+         ZP5mNz3Ka/ywpCGoq8rdgnXc7NunnkWeaBpYY/BWOmLU4WNXX8zS7etXXhQE4YPQEJT4
+         Sh2o/YIIjDLncJFMyE+25n3tbd2mIoLt4sjaCHE5ibm9w7zojyHM+LDCQ37cM74FEAAa
+         88KTn0gSnCFBCfojhfxOH78CpySHG3FFfTlpCefwP2A5J9MQlb6QdSVa9STYSx3IntJ4
+         L7Tg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=pBL4/bKUDw5E2zo1uR2Tl69h2iTlMgIAcnzQgodPCbU4jZ9kH+F5H9rfbzXCjT06J7
+         L72SYpdfgc5fOwM4GhRcdYnyK3wiXQ8ugpL19nbYt2iWo/vRF3GidawXXDGb2GUYpkzX
+         1Mz531cy2/HOsmQbUQ7304KV+OUghtcg8eLNnFuhQch7n12Kk3yy3AOzjrLoktcdgIsy
+         /HxBjyut0Au+A2t6si+PVwTHvC647a0BioeV0tUYLigzu3/jgP9Hb8eRZaXTX5VC6iZi
+         9QMH/+rXp05IK7OpGWh22xDpeV8CDkQ2sLFaBhKxtJ+nYoerM64t8EJXBBsVQb18ojGz
+         pW/A==
+X-Gm-Message-State: AOAM5330q6kn/TKataMNEVigNfNdr/xii/PQgHXzJyMbwLvsETlNfLoy
+	1rM9JBIGrcHeEDRx4qhZfl5S4bircceU7c3i6Fyn2fRO
+X-Google-Smtp-Source: ABdhPJwysG+S90b/g+9mK7LgeHhmJTBowst6JMhL16+a0coTi7P1NVp9jjaNHJfhvhLodYG6eHIvWdbQGJnAP2brEzI=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990137jab.84.1591791475066;
+ Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+MIME-Version: 1.0
+From: Deltachat Test <alice@gmail.com>
+Date: Wed, 10 Jun 2020 14:18:26 +0200
+Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+Subject: test
+To: bob@example.org, assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000006d8d7d05a7b9d5b3"
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/plain; charset="UTF-8"
+
+test
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">test<br></div>
+
+--0000000000006d8d7d05a7b9d5b3--
+
+--00000000000074432a05a7b9d512--

test-data/message/gmx_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Received: from mout-bounce.gmx.net ([212.227.15.44]) by mx-ha.gmx.net
+ (mxgmx101 [212.227.17.5]) with ESMTPS (Nemesis) id 1Mr97m-1jC6Y01o86-00oEqk
+ for <alice@gmx.de>; Tue, 09 Jun 2020 14:35:30 +0200
+Received: from localhost by mout-bounce.gmx.net id 0LhiZF-1jDTj11ZoH-00msO3
+    Tue, 09 Jun 2020 14:35:30 +0200
+Date: Tue, 09 Jun 2020 14:35:30 +0200
+From: "GMX Mailer Daemon" <mailer-daemon@gmx.de>
+To: alice@gmx.de
+Subject: Mail delivery failed: returning message to sender
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+X-UI-Out-Filterresults: unknown:0;V03:K0:O8yx6kuPaGQ=:0wIDPNXEr0wX2oNsLnXaWA
+ ==
+Envelope-To: <alice@gmx.de>
+X-GMX-Antispam: 0 (Mail was not recognized as spam); Detail=V3;
+X-Spam-Flag: NO
+X-UI-Filterresults: notjunk:1;V03:K0:QcE43EBhMmU=:IC5vvzi9jhPS/698Wuubzw1Q4N
+ h87X9j9B3CBN0ZKXB67KepwyNHmh9pxmFIUOMimylv7UK9np+j3X55roOd0nX9BmaaZ3Twvqf
+ UaSsxmyU+cNr6m3+oOb3udJBLe2pJEZDk1cOwACb5NXzYPSaIj4APfGCyvrzIx3FGkNuScNBb
+ tCbbKUJ0GB/VmJLB34XfF6dNN+Iwv9IQ9Yrvw/VXv9vWKsi3qRGGUt3yRw5jUKhQlBY21Pnoq
+ m0LqoMbAKfH1tKEQ/5TymH1ei50YKyWzZ89ISkQwkbYLaqN+6meGACpY18j43VCU9Fk4WQR7y
+ 3XvBYh2CO0CnCn+M9VsnasYag2sNrySe9nzyKfRTaxEg8qlJtl7kS4GX/FsxhHPavkqnU62Gl
+ 9V5TxIG7tmIR0Bf11sPzG/WGegoOHxrfz+qYR81llLMOHznpdDRKjsYDtO/rFBGZzYTiCZsrW
+ dZPVXV25SVcrDGZOaop3JoCbglmXLcSLLhmfE5MzyJEGte3I+6EiZJNeIe8qN3wMDTsRtJL9S
+ J6b2F/5/kTGVAWnXtNlf69BholCrxvjC4Snt3Xjc+7WIO8iw2c5YjmWy+4bAwd4uWll529hZd
+ 6pUYGwjFRnKleivCaJIt7DqbvbE7GZSbQH8fXm3zYqYTrrxiWdlykdoOA1OGbeM06RHJt3mJB
+ osZPU8BZKt0OiBOW64vg6gyAsNC0f02EA7dvRWYgFYqlSogfWZQIOKDKibMVHpIaA0foXg4BG
+ TEQDlsTIL0n2WC9WVqkMdm6xUXHgpArCrAsUhw3mEqPywEfJeBHn60tP2vQ9+pDIQAj5dQCDV
+ y96qSiCX4p31HfrWwAXB9mHfl4OO/tPcKUGBclj2rZ/NMc4O+7yDedLWXQnRzQExfOJLBbBh3
+ xgiNlWFHvDLn0pKG9EI1+3wJ7m2GF2jzDtbQTBv9z26DuAq5WbHZHupzeyfP7VCVXcKuB6sG1
+ 3+LWcdYtcXfqT58HwcvDLwowC4uJpiHfHwtVdiGMtHnmYLysp0V425g+vofQfNzBgR3d9JC15
+ G+HS44o6x6Legm6KnHYH3k0KhR7fgcgswJv/S+I/ryppUhGb2jezVZIUzgvAplzIUDAWnrHdF
+ KVqZ5wBJ0acShIfgMlsIxnBmcnIQ4R4jq3zAyj4XTFxVUFanU8ySiXubxV5PzJqj+GsVa2sjx
+ 9n/xQRJLwgMC4BYqzP6lEPwg/g5AneDAnl7ZlcQPC4SCMblC8N1KZyyIDTXPOI/o4lfdMYb9P
+ 7DmBp2S8aA2yuDe5XT20OmX3kVWeBOsBaAGvVFpIn7gwIDqnFh9WSMP6mkCwfChN3D1yLquYB
+ KAODgRZV5lVNmK+eOjW8m2oiRxmfxrjXLtw5PEhn3RkiRN4HnoePJeoYC7SG4EUwg+wYPu3M6
+ exP/YigoE6bjuBS5d0imUTRDMiwg469GyrFo1J1GkRVvj3lXSF4Nt11j6waqu1l0ReDYfU+QM
+ EMPGLEh1vRChCaqz4L7YI5FlSAVXxfmst0JRyE4k5r9CToEbZuYlPQ+jbcvptwBSaqzMb/YfT
+ cCrU2RWHUfmYIy8x8A/t8ScRbYPzs1lTK3yn1hYeXpw8Fgkip6DIIXAJwUUp+2SLcELICIo+p
+ uumMN0P/OZHH3V/hZ0dPr9xsYi7gdd/vyRIRPUwiL1rSp2WJGi+w4atun8kQBgnbAznRObDh1
+ 4zzKhApX9jo5gtFN6640QDEI5KpsMuPoty4rj9OK163ntKWGR451n+5ZX2FilTlpZYlIPO9Hy
+ SrjHzBog4texceR1OLh4pb/WFB0XFSjQchPAltXCYQFs82aDDk/A0nOPk=
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of
+its recipients. This is a permanent error. The following address(es)
+failed:
+
+snaerituhaeirns@gmail.com:
+SMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please
+ try
+550-5.1.1 double-checking the recipient's email address for typos or
+550-5.1.1 unnecessary spaces. Learn more at
+550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21
+9 - gsmtp
+
+
+
+--- The header of the original message is following. ---
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=gmx.net;
+	s=badeba3b8450; t=1591706130;
+	bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=X-UI-Sender-Class:To:From:Subject:Date;
+	b=NwY6W33mI1bAq6lpr6kbY+LD2hO9cDJBItTgY3NRIT94A6rKTVlSmhFM3AxYgFnj0
+	 Db0hncsNRDqcdtRoOo8Emcah5NJURvEQohG37lkug3GqneB4+FNTdYCeQbOKlZn6on
+	 pYYD/T9CmeL2HG3+8voeBjZIUenyXrF2WXG37hFY=
+X-UI-Sender-Class: 01bb95c1-4bf8-414a-932a-4f6e2808ef9c
+Received: from [192.168.178.30] ([84.57.126.154]) by mail.gmx.com (mrgmx005
+ [212.227.17.190]) with ESMTPSA (Nemesis) id 1MKbkM-1jNoq60HKm-00KyL2 for
+ <snaerituhaeirns@gmail.com>; Tue, 09 Jun 2020 14:35:30 +0200
+To: snaerituhaeirns@gmail.com
+From: Alice <alice@gmx.de>
+Subject: test
+Message-ID: <9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de>
+Date: Tue, 9 Jun 2020 14:36:10 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+X-Provags-ID: V03:K1:7awoptmynaF5MqxCAincdsa7zFCwQFNkFb6xRxigK06uEiGN00b
+ Mv2wJU91CEd4mvCCWzrTtaWLZDLH8pjAWaT4+HvYIUbpwNx2jC6WHTppkYYRMVJnm4lG9pr
+ SUx1OlIcp0kbsnl3mB9xYNFwm9jzpR9Kx8QEHwIbZiiSFBcH56498UGQi//kKXVMos8C14o
+ I7cmwYmr8xB09DwLMKXfg==
+X-Spam-Flag: NO
+X-UI-Out-Filterresults: notjunk:1;V03:K0:NWQAUbhAkBc=:yAaolOnVCDWEZhgUwwvtEs
+ wXbSJ/GfMvDRpCkYpFBvHXOTpGm6hjdjQ0vLK2hvu/Hz22UdlWbIdc1J2oO9S5U20mIdc+1bS
+ TPSSpqPFc7ICPx4Wbvv2SEp9ZqH2q7ORC52UvUWfI6OjAJEPDNrXQFdUiZAa72hLj1NPeG6Qi
+ 4AbL2HwLfJ8s6TeOCm6TXRRuD+w1o/ASFOqQmoao2dFyZ2BaoAgOKPKxXYfwVGceuUygpchyS
+ 0d2bZYOXSLR+6rUYevjZAq1OCi9AIC6/wlkOe5yIRk4gJFMfPauaICsdnq3uZ9ikCAX83VWun
+ PJVMxTLTP54lgo2h0jMBX3uKk10+/wzXWplllxX9NnSa3x1V28n6raslNF0IoC6Pm72kC6Jzr
+ GkC22viCm3/Y4uHlPMOXbY5WFrQe/D9GKeJeXBLoGciNwIFkUG12a+iqWtoT+h5HVObTW8LxM
+ +UtEl97nAwxYSM+sGfIpasRpZc7r/SgN3JWGO9R9WaXpW4Cc1dH7RI+hzuZDsDUBEGTUTVPDo
+ 0SvjKHiJ6sUqGDyfv4HUgVutus6EYP27LALND4ekfom2DPRFopZhbtV5fZT7CL1Q1NogU7tYf
+ /FdmR1T1J1zCAZSFvyR5LBkfglZlHzgdnTF2heuxyqKq4dm0hnLFSULB1+CVWsg8hzrruvO5q
+ XzA6qIhBQUZmWo7wBpqpkBPxzjgTGGtXc5y5e6+crxYbbuQdnUWEnyw2xI4d6pJPqtHDA2/vT
+ ZgvNDUGceavTR5Rtyb14hhX4Q6dWK3ATy16j4hs9Aq+q/IKyVAX3A5nFYbJRIz+2YnoLr2YOa
+ IrScEorXjvTxjw+aBy73SZBe2REPzJ+O6k7chVrYjV9Q28FiGVuRYJYxWw/59Pes7IAbmfQBV
+ 4vqGCQQr4eG78gVwjw+SNdp4/6jdNkIHDqR4XW9id/r5wYxKKj4UUkSor3/+h9Rd9srh+GApy
+ uOxw/ejFvbRcxFIjvadpq1KLnO7nM27nJ4lp44ul3i7VUGefLM/45TCsuds2HM1iQWhPFQ54y
+ SA5sYjf73EUJdkHchaf5i+4uSOmbOWQ4Yvmd8+IoyoXAxvEzY2Xh53nWi8ZPY1Tu4Bw8GRrz7
+ L+VK0QiWCg3/hM7wRlFFyshmMrScMk5fOf9ynqd0JbHB7u+n4/GUwx3im/w8+NgSd3YOz7wNU
+ KD1snDWoMUO8f23Ik1Osym688OLWNwKYT+mZbMIMXcz1fB+olRZn4czMhN5DiSb8hyOxRI8NE
+ PNfaoN87CXiRkgazV6U1eiRkfcK2AvI7zOJF1tclUHZ9awyYoXtxfEzZ+J/2TCXiC7V2iSkUF
+ EjwgPxlJmccccjsxc46v1ajnTxLo0tJbZ0+DJXWkCgQ0d/iiScQ=
+
+

test-data/message/posteo_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Delivered-To: alice@posteo.org
+Received: from proxy02.posteo.name ([127.0.0.1])
+	by dovecot03.posteo.local (Dovecot) with LMTP id zvCFJRzX317LGQIA+3EWog
+	for <alice@posteo.org>; Tue, 09 Jun 2020 20:44:24 +0200
+Received: from proxy02.posteo.de ([127.0.0.1])
+	by proxy02.posteo.name (Dovecot) with LMTP id mhNkNAnR316xBQMAGFAyLg
+	; Tue, 09 Jun 2020 20:44:23 +0200
+Received: from mailin06.posteo.de (unknown [10.0.1.6])
+	by proxy02.posteo.de (Postfix) with ESMTPS id 49hJtv3RRcz11m7
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+Received: from mx04.posteo.de (mailin06.posteo.de [127.0.0.1])
+	by mailin06.posteo.de (Postfix) with ESMTPS id 6935920DD2
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+X-Virus-Scanned: amavisd-new at posteo.de
+X-Spam-Flag: NO
+X-Spam-Score: -1
+X-Spam-Level: 
+X-Spam-Status: No, score=-1 tagged_above=-1000 required=8
+	tests=[ALL_TRUSTED=-1] autolearn=disabled
+Received: from mout01.posteo.de (mout01.posteo.de [185.67.36.65])
+	by mx04.posteo.de (Postfix) with ESMTPS id 49hJtv001Vz10kT
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Authentication-Results: mx04.posteo.de; dmarc=none (p=none dis=none) header.from=mout01.posteo.de
+Received: by mout01.posteo.de (Postfix)
+	id DCB6B1200DD; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+From: MAILER-DAEMON@mout01.posteo.de (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@posteo.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="B39111200B9.1591728262/mout01.posteo.de"
+Content-Transfer-Encoding: 7bit
+Message-Id: <20200609184422.DCB6B1200DD@mout01.posteo.de>
+
+This is a MIME-encapsulated message.
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Notification
+Content-Type: text/plain; charset=us-ascii
+
+This is the mail system at host mout01.posteo.de.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550
+    Requested action not taken: mailbox unavailable (in reply to RCPT TO
+    command)
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mout01.posteo.de
+X-Postfix-Queue-ID: B39111200B9
+X-Postfix-Sender: rfc822; alice@posteo.org
+Arrival-Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+
+Final-Recipient: rfc822; hanerthaertidiuea@gmx.de
+Original-Recipient: rfc822;hanerthaertidiuea@gmx.de
+Action: failed
+Status: 5.0.0
+Remote-MTA: dns; mx01.emig.gmx.net
+Diagnostic-Code: smtp; 550 Requested action not taken: mailbox unavailable
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Undelivered Message Headers
+Content-Type: text/rfc822-headers
+
+Return-Path: <alice@posteo.org>
+Received: from mout01.posteo.de (unknown [10.0.0.65])
+	by mout01.posteo.de (Postfix) with ESMTPS id B39111200B9
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Received: from submission-encrypt01.posteo.de (unknown [10.0.0.75])
+	by mout01.posteo.de (Postfix) with ESMTPS id 8A684160060
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=posteo.de; s=2017;
+	t=1591728262; bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=To:From:Subject:Date:From;
+	b=brJnt4PLAX3Tda1RHCo91aB1kMAL/Ku9dmO7D2DD41Zu5ShNsyqqyDkyxb1DsDn3O
+	 6KuBZe3/8gemBuCJ/mxzwd9v8sBnlrV+5afIk0Ye9VvthZsc4HoG79+FiVOi9F38o0
+	 DtJJFYFw/X7mAc5Xyt0B0JvtiTPpBdRAkluUQm+QW6cW6GGlwicVW19qvebzq+sHyP
+	 X2bZ8wpo78yVgvjPBK3DLaXa+pKFMBjLdDUcIE2bZnY6u6F1x8SXGKGBoxVwdJipJx
+	 v14so5IejNsf4LYJjH3Qb8xgK1aAi6e6nQn4YXV0INL6ahzgALiT9N6vwunNKYVJNi
+	 fPPKvBWDfUS4Q==
+Received: from customer (localhost [127.0.0.1])
+	by submission (posteo.de) with ESMTPSA id 49hJtt1WPbz6tmV
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+To: hanerthaertidiuea@gmx.de
+From: deltachat <alice@posteo.org>
+Subject: test
+Message-ID: <04422840-f884-3e37-5778-8192fe22d8e1@posteo.de>
+Date: Tue, 9 Jun 2020 20:45:02 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+Posteo-User: alice@posteo.org
+Posteo-Dkim: ok
+
+--B39111200B9.1591728262/mout01.posteo.de--

test-data/message/testrun_ndn.eml

@@ -0,0 +1,107 @@
+Return-Path: <>
+Delivered-To: alice@testrun.org
+Received: from hq5.merlinux.eu
+	by hq5.merlinux.eu (Dovecot) with LMTP id Ye02K6PB5F43cQAAPzvFDg
+	for <alice@testrun.org>; Sat, 13 Jun 2020 14:08:03 +0200
+Received: by hq5.merlinux.eu (Postfix)
+	id 9EBE627A0B2E; Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+Date: Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+From: MAILER-DAEMON@hq5.merlinux.eu (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@testrun.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="CDB8D27A0B2C.1592050083/hq5.merlinux.eu"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20200613120803.9EBE627A0B2E@hq5.merlinux.eu>
+
+This is a MIME-encapsulated message.
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host hq5.merlinux.eu.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1
+    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in
+    virtual mailbox table (in reply to RCPT TO command)
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Delivery report
+Content-Type: message/global-delivery-status
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; hq5.merlinux.eu
+X-Postfix-Queue-ID: CDB8D27A0B2C
+X-Postfix-Sender: rfc822; alice@testrun.org
+Arrival-Date: Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+
+Final-Recipient: rfc822; hcksocnsofoejx@five.chat
+Original-Recipient: rfc822;hcksocnsofoejx@five.chat
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; mail.five.chat
+Diagnostic-Code: smtp; 550 5.1.1 <hcksocnsofoejx@five.chat>: Recipient address
+    rejected: User unknown in virtual mailbox table
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Undelivered Message
+Content-Type: message/global
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <alice@testrun.org>
+Received: from localhost (p200300edb723070079835ce22985a199.dip0.t-ipconnect.de [IPv6:2003:ed:b723:700:7983:5ce2:2985:a199])
+	by hq5.merlinux.eu (Postfix) with UTF8SMTPSA id CDB8D27A0B2C;
+	Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org;
+	s=testrun; t=1592050082;
+	bh=Kvhta0OMsTRVC7OlaAqo68TBE0KuGBv4vUBp6Ez/7VY=;
+	h=Subject:References:In-Reply-To:Date:To:From:From;
+	b=Ql60JEGFXLNvjsyihATw2z34ct++8xZvTPNw0snXe6+oqdqsRZJ9tWNDTxOgx8Iqf
+	 HQ4puBVGcWjIlszYQVLlq3APi04o2ep3GrD8EF0J0GpDdW8yw6wCos6Q8r+TWmXwET
+	 kGXHTRPVaUIqZF2i/utypxMfd1ua0S3jBDnIXTe/p2+XvfC3Cf3hZGW+FQ/Zd7G8Vh
+	 /U2rgX5BTIGf26ZCbmcMaXWkftgv6+yns0AmzorV9yB+EhTkWIUjk+C25bRtMbJ5mZ
+	 93dwdr+sXrrSZLSi+LBqc57Dv9j4p/SUmB4zPlvfUv7/bqLi36pypvtCJ5Ul8UEXSb
+	 XNFZPaEl+mwjA==
+Content-Type: text/plain; charset=utf-8
+Chat-Disposition-Notification-To: alice@testrun.org
+Subject: =?utf-8?q?Message_from_hocuri1=40testrun=2Eorg?=
+MIME-Version: 1.0
+References: <Mr.VSg3KXFUOTG.9sn7JBxZn1W@testrun.org>
+	<Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+In-Reply-To: <Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+Date: Sat, 13 Jun 2020 12:08:01 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@testrun.org; prefer-encrypt=mutual;
+	keydata=xjMEXt3z1xYJKwYBBAHaRw8BAQdAf6MctU/8cmEqwEN9VFZ3gHBFIxKiEaARZl1DFUkI7e
+	rNFTxob2N1cmkxQHRlc3RydW4ub3JnPsKLBBAWCAAzAhkBBQJe3fPXAhsDBAsJCAcGFQgJCgsCAxYC
+	ARYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygAG2IA/1nTmmmkHAc1Bjtx2FOstbaS+N
+	XHjxaK+hkoWllsyhz0AQDJJ1++u7jVZPRn/j1LlByrT3Jv/D1aY14J5rjj+ADVBM44BF7d89cSCisG
+	AQQBl1UBBQEBB0DpSTaZ30dAVwM9PkBe2h+gFyxn9HSorP4XCHJu/lIdPAMBCAfCeAQYFggAIAUCXt
+	3z1wIbDBYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygA2QkA/16toWCtseYKw8G1X2j7
+	xYR3Cyabq37hgbesDOThIIzNAP0UCUS8mnunmkS5adEbftRaDi2JZoGxDw46jtJJ2+13Cw==
+Message-ID: <Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org>
+To: <hcksocnsofoejx@five.chat>
+From:  <alice@testrun.org>
+
+F
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu--

test-data/message/tiscali_ndn.eml

@@ -0,0 +1,91 @@
+Return-Path: <>
+Delivered-To: alice@tiscali.it
+Received: from director-5.mail.tiscali.sys ([10.39.80.174])
+	by dovecot-08.mail.tiscali.sys with LMTP id SBRfEpGb517VAgAAd2fHbg
+	for <alice@tiscali.it>; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from cmgw-4.mail.tiscali.it ([10.39.80.174])
+	by director-5.mail.tiscali.sys with LMTP id MFUPL5Cb516tawAArQJVuQ
+	; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from michael.mail.tiscali.it ([213.205.33.246])
+	by cmgw-4.mail.tiscali.it with 
+	id rTtS2200V5JdeUd01U2RlV; Mon, 15 Jun 2020 16:02:25 +0000
+x-cnfs-analysis: v=2.3 cv=ZdPMyfdA c=1 sm=1 tr=0 cx=a_idp_d
+ a=AfTPebshMYb+aQOCLa9q3Q==:117 a=HpEJnUlJZJkA:10 a=jmdcTMp_Gj4A:10
+ a=r77TgQKjGQsHNAKrUKIA:9 a=b8iBRs35AAAA:8 a=NMdB-582e605uxHDr_AA:9
+ a=QEXdDO2ut3YA:10 a=MhhPCb74-dYA:10 a=HXlsH_Kov2KnitTn7A4A:9
+ a=BkuCPOF3BOzethIN9HQA:9 a=qG5HpJ6ZyD35YNEB:21 a=kvHihYffoorsyJbA:21
+ a=xD8EQi6zkreDqSNPYj5l:22
+Date: Mon, 15 Jun 2020 16:02:25 +0000
+From: Mail Delivery System <mail-daemon@smtp.tiscali.it>
+To: alice@tiscali.it
+Subject: Delivery status notification
+MIME-Version: 1.0
+Content-Type: multipart/report; boundary="------------I305M09060309060P_896715922369450"
+
+This is a multi-part message in MIME format.
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/plain; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+      This is an automatically generated Delivery Status Notification.      
+
+Delivery to the following recipients was aborted after 2 second(s):
+
+  * shenauithz@testrun.org
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: message/delivery-status; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; michael.mail.tiscali.it [213.205.33.13]
+Received-From-MTA: dns; localhost [146.241.100.150]
+Arrival-Date: Mon, 15 Jun 2020 16:02:23 +0000
+
+
+Final-recipient: rfc822; shenauithz@testrun.org
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550 5.1.1 <shenauithz@testrun.org>: Recipient address rejected: User unknown in virtual mailbox table
+Last-attempt-Date: Mon, 15 Jun 2020 16:02:25 +0000
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/rfc822-headers; Content-Transfer-Encoding: 8bit
+Content-Disposition: attachment
+
+x-auth-user: alice@tiscali.it
+Chat-Disposition-Notification-To: alice@tiscali.it
+Chat-User-Avatar: avatar.jpg
+Subject: =?utf-8?q?Message_from_=F0=9F=8F=9E=EF=B8=8F_Mefiscali?=
+MIME-Version: 1.0
+Date: Mon, 15 Jun 2020 16:02:22 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@tiscali.it; prefer-encrypt=mutual;
+	keydata=xjMEXtFRUBYJKwYBBAHaRw8BAQdA5sqHJqkWlveCgsNd0rtwtZrT1mmo1gwaGC5+WheYk5
+	nNHTxhbmRyZWFzLmxhdHRtYW5uQHRpc2NhbGkuaXQ+wosEEBYIADMCGQEFAl7RUXoCGwMECwkIBwYV
+	CAkKCwIDFgIBFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrAAaAEA4wssXeU2IXnowv
+	iu3zmcNzDgE4HdmW4RFyqJC6bgxXQA/3aTfE/PhQgZvi6RrKMvP4zygXpD9y+3ydIZP88Bp8kIzjgE
+	XtFRUBIKKwYBBAGXVQEFAQEHQKaAwlP0j9m0aYsCtO+qD9+foH0kiTN5BWDe5YcZrckVAwEIB8J4BB
+	gWCAAgBQJe0VF6AhsMFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrA1JgEAkscCQlps
+	h3ZxlLqBlbf2+85f4S4aGQfFPtIYEkKKhYEBAJbQulNNp9UarvhfyBiIdvkBVDcCnJZwzbORqp8RM0 gC
+Message-ID: <Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it>
+To: <shenauithz@testrun.org>
+From: =?utf-8?b?8J+Pnu+4jyBNZWZpc2NhbGk=?= <alice@tiscali.it>
+Content-Type: multipart/mixed; boundary="5uAmYQux1HZxxriijTjjKSp4DMoJwq"
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tiscali.it; s=smtp;
+	t=1592236943; bh=C3taz+zuSre1ko5Q5CzGPmbyrgegKYBClx/3Dv7t/Xw=;
+	h=Subject:Date:To:From;
+	b=LrcfLfrQoemOkHTQsqR8MExqNlx5KPYNFWhwlBWylvVc5GlmlhzqM6SAVKd0NVsKE
+	 gVRlBId5FvnlwoJ2WZnXaw/+3lWKilMTuzzQ1oFGvLnZ1XUaUEfuliIv+9NI79dJWX
+	 +S3jsSgzJMJc9+fO6s9bJsX1EHQ2a8GXNbwDtLXs=
+
+
+
+--------------I305M09060309060P_896715922369450--
+
+