fix warnings and bugs, noticed on nightly

prepare 1.36

.circleci/config.yml

@@ -138,12 +138,6 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests_rust:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_rust.sh
-    
   remote_tests_python:
     machine: true
     steps:
@@ -178,11 +172,6 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
-
       - remote_tests_python:
           filters:
             tags:
@@ -191,8 +180,9 @@ workflows:
       - remote_python_packaging:
           requires: 
              - remote_tests_python
-             - remote_tests_rust 
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 
@@ -201,6 +191,8 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -212,6 +204,8 @@ workflows:
 
       - build_doxygen:
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: Rust CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.43.1
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.43.1
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly, 1.43.1]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      with:
+        command:  check
+        args: --workspace --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --workspace
+          

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-19
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

CHANGELOG.md

@@ -1,5 +1,50 @@
 # Changelog 
 
+## 1.36.0
+
+- parse ndn (network delivery notification) reports
+  and report faild messages as such #1552 #1622 #1630
+
+- read image orientation from exif before recoding #1619
+
+- improve logging #1593 #1598
+
+- improve python and bot bindings #1583 #1609
+
+- improve imap logout #1595
+
+- fix sorting #1600 #1604
+
+- fix qr code generation #1631
+
+- update rustcrypto releases #1603
+
+- refactorings #1617
+
+
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
 ## 1.34.0
 
 - new api for io, thread and event handling #1356,

Cargo.toml

@@ -1,23 +1,23 @@
 [package]
 name = "deltachat"
-version = "1.34.0"
+version = "1.36.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
 [profile.release]
-# lto = true
+lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
 async-smtp = { version = "0.3" }
@@ -25,42 +25,43 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.3.1"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.0", features = ["unstable"] }
-base64 = "0.11"
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
+kamadak-exif = "0.5"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.22", features = ["bundled"] }
-r2d2_sqlite = "0.15.0"
+rusqlite = { version = "0.23", features = ["bundled"] }
+r2d2_sqlite = "0.16.0"
 r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
+strum = "0.18.0"
+strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.17.1"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
 mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
+async-std-resolver = "0.19.5"
 
-pretty_env_logger = { version = "0.3.1", optional = true }
+pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
@@ -69,8 +70,8 @@ ansi_term = { version = "0.12.1", optional = true }
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
 async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
 smol = "0.1.10"
 
@@ -98,5 +99,3 @@ repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 
-[patch.crates-io]
-smol = { git = "https://github.com/dignifiedquire/smol-1", branch = "isolate-nix" }

appveyor.yml

@@ -1,19 +0,0 @@
-environment:
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-19
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - rustc -vV
-  - cargo -vV
-
-build: false
-
-test_script:
-  - cargo test --release --all
-
-cache:
-  - target
-  - C:\Users\appveyor\.cargo\registry

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.34.0"
+version = "1.36.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -307,9 +307,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    DC_MEDIA_QUALITY_WORSE (1)
  *                    allow worse images/videos/voice quality to gain smaller sizes,
  *                    suitable for providers or areas known to have a bad connection.
- *                    In contrast to other options, the implementation of this option is currently up to the UIs;
- *                    this may change in future, however,
- *                    having the option in the core allows provider-specific-defaults already today.
+ *                    The library uses the `media_quality` setting to use different defaults
+ *                    for recoding images sent with type DC_MSG_IMAGE.
+ *                    If needed, recoding other file types is up to the UI.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -528,9 +528,20 @@ int             dc_is_io_running(const dc_context_t* context);
 void            dc_stop_io(dc_context_t* context);
 
 /**
- * This function can be called whenever there is a hint
- * that the network is available again.
- * The library will try to send pending messages out.
+ * This function should be called when there is a hint
+ * that the network is available again,
+ * eg. as a response to system event reporting network availability.
+ * The library will try to send pending messages out immediately.
+ *
+ * Moreover, to have a reliable state
+ * when the app comes to foreground with network available,
+ * it may be reasonable to call the function also at that moment.
+ *
+ * It is okay to call the function unconditionally when there is
+ * network available, however, calling the function
+ * _without_ having network may interfere with the backoff algorithm
+ * and will led to let the jobs fail faster, with fewer retries
+ * and may avoid messages being sent out.
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
@@ -754,6 +765,15 @@ uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t ch
  * dc_msg_unref(msg);
  * ~~~
  *
+ * If you send images with the DC_MSG_IMAGE type,
+ * they will be recoded to a reasonable size before sending, if possible
+ * (cmp the dc_set_config()-option `media_quality`).
+ * If that fails, is not possible, or the image is already small enough, the image is sent as original.
+ * If you want images to be always sent as the original file, use the DC_MSG_FILE type.
+ *
+ * Videos and other file types are currently not recoded by the library,
+ * with dc_prepare_msg(), however, you can do that from the UI.
+ *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id Chat ID to send the message to.
@@ -2782,6 +2802,9 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_DELIVERED.
  * - DC_STATE_OUT_MDN_RCVD (28) - Outgoing message read by the recipient (two checkmarks; this requires goodwill on the receiver's side)
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_READ.
+ *   Also messages already read by some recipients
+ *   may get into the state DC_STATE_OUT_FAILED at a later point,
+ *   eg. when in a group, delivery fails for some recipients.
  *
  * If you just want to check if a message is sent or not, please use dc_msg_is_sent() which regards all states accordingly.
  *
@@ -4117,8 +4140,9 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
- * DC_STATE_OUT_FAILED, see dc_msg_get_state().
+ * A single message could not be sent.
+ * State changed from DC_STATE_OUT_PENDING, DC_STATE_OUT_DELIVERED or DC_STATE_OUT_MDN_RCVD
+ * to DC_STATE_OUT_FAILED, see dc_msg_get_state().
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -4427,12 +4451,22 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_DEVICE_MESSAGES_HINT       70
 #define DC_STR_WELCOME_MESSAGE            71
 #define DC_STR_UNKNOWN_SENDER_FOR_CHAT    72
-#define DC_STR_COUNT                      72
+#define DC_STR_SUBJECT_FOR_NEW_CONTACT    73
+#define DC_STR_FAILED_SENDING_TO          74
+
+#define DC_STR_COUNT                      74
 
 /*
  * @}
  */
 
+#ifdef PY_CFFI_INC
+/* Helper utility to locate the header file when building python bindings. */
+char* _dc_header_file_location(void) {
+    return __FILE__;
+}
+#endif
+
 
 #ifdef __cplusplus
 }

deltachat-ffi/src/lib.rs

@@ -285,7 +285,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     }
     let ctx = &*context;
 
-    block_on({ ctx.start_io() })
+    block_on(ctx.start_io())
 }
 
 #[no_mangle]
@@ -295,7 +295,7 @@ pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c
     }
     let ctx = &*context;
 
-    block_on({ ctx.is_io_running() }) as libc::c_int
+    block_on(ctx.is_io_running()) as libc::c_int
 }
 
 #[no_mangle]
@@ -1829,7 +1829,11 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
     }
     let ctx = &*context;
 
-    block_on({ location::send_locations_to_chat(&ctx, ChatId::new(chat_id), seconds as i64) });
+    block_on(location::send_locations_to_chat(
+        &ctx,
+        ChatId::new(chat_id),
+        seconds as i64,
+    ));
 }
 
 #[no_mangle]

python/examples/echo_and_quit.py

@@ -12,7 +12,7 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
                 message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))

python/examples/test_examples.py

@@ -26,15 +26,15 @@ def test_echo_quit_plugin(acfactory, lp):
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
+    bot_chat = bot_contact.create_chat()
+    bot_chat.send_text("hello")
 
-    lp.sec("waiting for the bot-reply to arrive")
+    lp.sec("waiting for the reply message from the bot to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    assert reply.chat == ch1
     lp.sec("send quit sequence")
-    ch1.send_text("/quit")
+    bot_chat.send_text("/quit")
     botproc.wait()
 
 
@@ -47,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+    ac1.add_account_plugin(FFIEventLogger(ac1))
+    ac2.add_account_plugin(FFIEventLogger(ac2))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -60,7 +60,8 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = events.FFIEventLogger(ac, "bot")
+        ac.set_config("displayname", "bot")
+        log = events.FFIEventLogger(ac)
         ac.add_account_plugin(log)
 
     for plugin in account_plugins or []:
@@ -76,8 +77,8 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.configure()
-        ac.wait_configure_finish()
+        configtracker = ac.configure()
+        configtracker.wait_finish()
 
     # start IO threads and configure if neccessary
     ac.start_io()

python/src/deltachat/_build.py

@@ -1,65 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -71,13 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -213,22 +213,40 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, email, name=None):
-        """ create a (new) Contact. If there already is a Contact
-        with that e-mail address, it is unblocked and its name is
-        updated.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        :param email: email-address (text type)
-        :param name: display name for this contact (optional)
+        Calling this method will always resulut in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        `name` is updated if specified.
+
+        :param obj: email-address, Account or Contact instance.
+        :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
+        if isinstance(obj, Account):
+            if not obj.is_configured():
+                raise ValueError("can only add addresses from configured accounts")
+            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
+        elif isinstance(obj, Contact):
+            if obj.account != self:
+                raise ValueError("account mismatch {}".format(obj))
+            addr, displayname = obj.addr, obj.name
+        elif isinstance(obj, str):
+            displayname, addr = parseaddr(obj)
+        else:
+            raise TypeError("don't know how to create chat for %r" % (obj, ))
+
+        if name is None and displayname:
+            name = displayname
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -250,6 +268,13 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -279,53 +304,29 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+    def create_chat(self, obj):
+        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        return self.create_contact(obj).create_chat()
 
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(contact, "id"):
-            if contact.account != self:
-                raise ValueError("Contact belongs to a different Account")
-            contact_id = contact.id
-        else:
-            assert isinstance(contact, int)
-            contact_id = contact
-        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
-        return Chat(self, chat_id)
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
 
-    def create_chat_by_message(self, message):
-        """ create or get an existing chat object for the
-        the specified message.
-
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
-        :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(message, "id"):
-            if message.account != self:
-                raise ValueError("Message belongs to a different Account")
-            msg_id = message.id
-        else:
-            assert isinstance(message, int)
-            msg_id = message
-        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
-        return Chat(self, chat_id)
-
-    def create_group_chat(self, name, verified=False):
+    def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
+        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        return Chat(self, chat_id)
+        chat = Chat(self, chat_id)
+        if contacts is not None:
+            for contact in contacts:
+                chat.add_contact(contact)
+        return chat
 
     def get_chats(self):
         """ return list of chats.
@@ -354,13 +355,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -567,29 +561,24 @@ class Account(object):
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
-        :returns: None (account is configured and with io-scheduling running)
+        :returns: None
         """
         if not self.is_configured():
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
     def configure(self):
+        """ Start configuration process and return a Configtracker instance
+        on which you can block with wait_finish() to get a True/False success
+        value for the configuration process.
+        """
         assert not self.is_configured()
-        assert not hasattr(self, "_configtracker")
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
-        if hasattr(self, "_configtracker"):
-            self.remove_account_plugin(self._configtracker)
-        self._configtracker = ConfigureTracker()
-        self.add_account_plugin(self._configtracker)
+        configtracker = ConfigureTracker(self)
+        self.add_account_plugin(configtracker)
         lib.dc_configure(self._dc_context)
-
-    def wait_configure_finish(self):
-        try:
-            self._configtracker.wait_finish()
-        finally:
-            self.remove_account_plugin(self._configtracker)
-            del self._configtracker
+        return configtracker
 
     def is_started(self):
         return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))

python/src/deltachat/chat.py

@@ -18,6 +18,8 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -328,33 +330,34 @@ class Chat(object):
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, contact):
+    def add_contact(self, obj):
         """ add a contact to this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
+        return contact
 
-    def remove_contact(self, contact):
+    def remove_contact(self, obj):
         """ remove a contact from this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
-        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-
         """
         from .contact import Contact
         dc_array = ffi.gc(
@@ -365,6 +368,14 @@ class Chat(object):
             dc_array, lambda id: Contact(self.account, id))
         )
 
+    def num_contacts(self):
+        """ return number of contacts in this chat. """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return lib.dc_array_get_cnt(dc_array)
+
     def set_profile_image(self, img_path):
         """Set group profile image.
 

python/src/deltachat/const.py

@@ -1,197 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -3,6 +3,8 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
+from .chat import Chat
+from . import const
 
 
 class Contact(object):
@@ -11,6 +13,8 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -36,10 +40,13 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def display_name(self):
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
+    # deprecated alias
+    display_name = name
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -58,6 +65,16 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)
+    def create_chat(self):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        dc_context = self.account._dc_context
+        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
+        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
+        return Chat(self.account, chat_id)
+
+    # deprecated name
+    get_chat = create_chat

python/src/deltachat/direct_imap.py

@@ -0,0 +1,213 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
+import io
+import email
+import ssl
+import pathlib
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
+
+
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+ALL = "1:*"
+
+
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account):
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        ssl_context = ssl.create_default_context()
+
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
+        try:
+            self.conn.logout()
+        except (OSError, IMAPClientError):
+            print("Could not logout direct_imap conn")
+
+    def select_folder(self, foldername):
+        assert not self._idling
+        return self.conn.select_folder(foldername)
+
+    def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
+        if "_" not in config_name:
+            config_name = "configured_{}_folder".format(config_name)
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+
+    def list_folders(self):
+        """ return list of all existing folder names"""
+        assert not self._idling
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
+
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self):
+        assert not self._idling
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
+        assert not self._idling
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.set_flags(messages, [SEEN])
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self):
+        return len(self.get_unread_messages())
+
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[HEADER]']
+                flags = data[FLAGS]
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/events.py

@@ -28,13 +28,9 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
+    def __init__(self, account):
         self.account = account
-        self.logid = logid
+        self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
@@ -127,6 +123,12 @@ class FFIEventTracker:
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
 
+    def wait_msg_delivered(self, msg):
+        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == msg.chat.id
+        assert ev.data2 == msg.id
+        assert msg.is_out_delivered()
+
 
 class EventThread(threading.Thread):
     """ Event Thread for an account.

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -88,6 +88,14 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
+    @global_hookspec
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -53,15 +53,19 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
+    def create_chat(self):
+        """ create or get an existing chat (group) object for this message.
+
+        If the message is a deaddrop contact request
+        the sender will become an accepted contact.
+
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
-                lib.dc_msg_unref
-        )
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):

python/src/deltachat/testplugin.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import sys
+import io
 import subprocess
 import queue
 import threading
@@ -16,6 +17,7 @@ from . import Account, const
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
@@ -33,9 +35,6 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
@@ -216,6 +215,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -226,13 +226,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
             ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+                ac.add_account_plugin(FFIEventLogger(ac))
             self._accounts.append(ac)
             return ac
 
@@ -301,31 +303,27 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1.wait_configure_finish()
-            ac1.start_io()
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1.wait_configure_finish()
-            ac1.start_io()
-            ac2.wait_configure_finish()
-            ac2.start_io()
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True, quiet=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
+            self.wait_configure_and_start_io()
             for acc in accounts:
-                acc._configtracker.wait_finish()
-                acc.start_io()
+                acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
@@ -343,14 +341,28 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
                 "-u",
@@ -375,9 +387,42 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    return am
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
@@ -445,4 +490,17 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
+
     return Printer()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # set a report attribute for each phase of a call, which can
+    # be "setup", "call", "teardown"
+
+    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -40,12 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -59,7 +61,10 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
+        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """

python/tests/stress_test_db.py

@@ -110,7 +110,7 @@ class AutoReplier:
         if self.current_sent >= self.num_send:
             self.report_func(self, ReportType.exit)
             return
-        message.accept_sender_contact()
+        message.create_chat()
         message.mark_seen()
         self.log("incoming message: {}".format(message))
 

python/tests/test_increation.py

@@ -36,9 +36,7 @@ def wait_msgs_changed(account, msgs_list):
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -48,9 +46,7 @@ class TestOnlineInCreation:
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -64,9 +60,7 @@ class TestOnlineInCreation:
     def test_forward_increation(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
@@ -80,7 +74,7 @@ class TestOnlineInCreation:
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(c2)
+        chat2.add_contact(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them

python/tests/test_lowlevel.py

@@ -69,8 +69,8 @@ def test_sig():
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
 
-    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    chat = ac1.create_chat_by_contact(contact1)
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]

python/tox.ini

@@ -71,6 +71,8 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
+markers = 
+    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

spec.md

@@ -1,10 +1,12 @@
-# Chat-over-Email specification
+# chat-mail specification
 
-Version 0.30.0
+Version: 0.32.0
+Status:  In-progress 
+Format:  [Semantic Line Breaks](https://sembr.org/)
 
-This document describes how emails can be used
-to implement typical messenger functions
-while staying compatible to existing MUAs.
+This document roughly describes how chat-mail 
+apps use the standard e-mail system 
+to implement typical messenger functions.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -30,17 +32,14 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
-If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD start with the characters `Chat:`
-and SHOULD be an excerpt of the message.
+the `Subject` header SHOULD be `Message from <sender name>`.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -58,7 +57,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Chat: Hello ...
+    Subject: Message from sender@domain
 
     Hello world!
 

src/blob.rs

@@ -8,11 +8,15 @@ use async_std::prelude::*;
 use async_std::{fs, io};
 
 use image::GenericImageView;
+use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::constants::AVATAR_SIZE;
+use crate::config::Config;
+use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::Event;
+use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -57,7 +61,7 @@ impl<'a> BlobObject<'a> {
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -372,11 +376,73 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
     }
+
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
+            != Some((Viewtype::Image, "image/jpeg"))
+        {
+            return Ok(());
+        }
+
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
+
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
+        }
+
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
+    }
+
+    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exifreader = exif::Reader::new();
+        let exif = exifreader.read_from_container(&mut bufreader)?;
+        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
+            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
+            // we only use rotation, in practise, flipping is not used.
+            match orientation.value.get_uint(0) {
+                Some(3) => return Ok(180),
+                Some(6) => return Ok(90),
+                Some(8) => return Ok(270),
+                other => warn!(context, "exif orientation value ignored: {:?}", other),
+            }
+        }
+        Ok(0)
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -400,7 +466,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {

src/chat.rs

@@ -39,18 +39,9 @@ impl ChatId {
         ChatId(id)
     }
 
-    /// A ChatID which indicates an error.
-    ///
-    /// This is transitional and should not be used in new code.  Do
-    /// not represent errors in a ChatId.
-    pub fn is_error(self) -> bool {
-        self.0 == 0
-    }
-
     /// An unset ChatId
     ///
-    /// Like [ChatId::is_error], from which it is indistinguishable, this is
-    /// transitional and should not be used in new code.
+    /// This is transitional and should not be used in new code.
     pub fn is_unset(self) -> bool {
         self.0 == 0
     }
@@ -431,24 +422,44 @@ impl ChatId {
     }
 
     async fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
-        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
-        self.parent_query(
-            context,
-            "rfc724_mid, mime_in_reply_to, mime_references",
-            collect,
-        )
-        .await
-        .ok()
-        .flatten()
+        let collect =
+            |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?));
+        let (rfc724_mid, mime_in_reply_to, mime_references, error): (
+            String,
+            String,
+            String,
+            String,
+        ) = self
+            .parent_query(
+                context,
+                "rfc724_mid, mime_in_reply_to, mime_references, error",
+                collect,
+            )
+            .await
+            .ok()
+            .flatten()?;
+
+        if !error.is_empty() {
+            // Do not reply to error messages.
+            //
+            // An error message could be a group chat message that we failed to decrypt and
+            // assigned to 1:1 chat. A reply to it will show up as a reply to group message
+            // on the other side. To avoid such situations, it is better not to reply to
+            // error messages at all.
+            None
+        } else {
+            Some((rfc724_mid, mime_in_reply_to, mime_references))
+        }
     }
 
     async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
-        let collect = |row: &rusqlite::Row| Ok(row.get(0)?);
-        let packed: Option<String> = self.parent_query(context, "param", collect).await?;
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?));
+        let res: Option<(String, String)> =
+            self.parent_query(context, "param, error", collect).await?;
 
-        if let Some(ref packed) = packed {
+        if let Some((ref packed, ref error)) = res {
             let param = packed.parse::<Params>()?;
-            Ok(param.exists(Param::GuaranteeE2ee))
+            Ok(error.is_empty() && param.exists(Param::GuaranteeE2ee))
         } else {
             // No messages
             Ok(false)
@@ -1341,6 +1352,12 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Er
             .ok_or_else(|| {
                 format_err!("Attachment missing for message of type #{}", msg.viewtype)
             })?;
+
+        if msg.viewtype == Viewtype::Image {
+            if let Err(e) = blob.recode_to_image_size(context).await {
+                warn!(context, "Cannot recode image, using original data: {:?}", e);
+            }
+        }
         msg.param.set(Param::File, blob.as_name());
 
         if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
@@ -1447,7 +1464,7 @@ pub async fn send_msg(
                 }
             }
             msg.param.remove(Param::PrepForwards);
-            msg.save_param_to_disk(context).await;
+            msg.update_param(context).await;
         }
         return send_msg_inner(context, chat_id, msg).await;
     }
@@ -1930,20 +1947,19 @@ pub async fn create_group_chat(
         .sql
         .get_rowid(context, "chats", "grpid", grpid)
         .await?;
-    let chat_id = ChatId::new(row_id);
-    if !chat_id.is_error() {
-        if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
-            let mut draft_msg = Message::new(Viewtype::Text);
-            draft_msg.set_text(Some(draft_txt));
-            chat_id.set_draft_raw(context, &mut draft_msg).await;
-        }
 
-        context.emit_event(Event::MsgsChanged {
-            msg_id: MsgId::new(0),
-            chat_id: ChatId::new(0),
-        });
+    let chat_id = ChatId::new(row_id);
+    if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
+        let mut draft_msg = Message::new(Viewtype::Text);
+        draft_msg.set_text(Some(draft_txt));
+        chat_id.set_draft_raw(context, &mut draft_msg).await;
     }
 
+    context.emit_event(Event::MsgsChanged {
+        msg_id: MsgId::new(0),
+        chat_id: ChatId::new(0),
+    });
+
     Ok(chat_id)
 }
 
@@ -2580,7 +2596,7 @@ pub async fn forward_msgs(
                         .set(Param::PrepForwards, new_msg_id.to_u32().to_string());
                 }
 
-                msg.save_param_to_disk(context).await;
+                msg.update_param(context).await;
                 msg.param = save_param;
             } else {
                 msg.state = MessageState::OutPending;
@@ -2768,7 +2784,7 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
 pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
 
-    if context.sql.execute(
+    if let Err(e) = context.sql.execute(
         "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid) VALUES (?,?,?, ?,?,?, ?,?);",
         paramsv![
             chat_id,
@@ -2780,7 +2796,8 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
             text.as_ref().to_string(),
             rfc724_mid,
         ]
-    ).await.is_err() {
+    ).await {
+        warn!(context, "Could not add info msg: {}", e);
         return;
     }
 

src/config.rs

@@ -11,7 +11,7 @@ use crate::dc_tools::*;
 use crate::events::Event;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::stock::StockMessage;
+use crate::{scheduler::InterruptInfo, stock::StockMessage};
 
 /// The available configuration keys.
 #[derive(
@@ -104,6 +104,9 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
+    ConfiguredInboxFolder,
+    ConfiguredMvboxFolder,
+    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -137,6 +140,7 @@ impl Context {
         // Default values
         match key {
             Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
@@ -199,17 +203,18 @@ impl Context {
             }
             Config::InboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_inbox(false).await;
+                self.interrupt_inbox(InterruptInfo::new(false, None)).await;
                 ret
             }
             Config::SentboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_sentbox(false).await;
+                self.interrupt_sentbox(InterruptInfo::new(false, None))
+                    .await;
                 ret
             }
             Config::MvboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_mvbox(false).await;
+                self.interrupt_mvbox(InterruptInfo::new(false, None)).await;
                 ret
             }
             Config::Selfstatus => {

src/configure/mod.rs

@@ -272,9 +272,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
         || ctx.get_config_bool(Config::MvboxMove).await;
 
-    imap.configure_folders(ctx, create_mvbox)
-        .await
-        .context("configuring folders failed")?;
+    imap.configure_folders(ctx, create_mvbox).await?;
 
     imap.select_with_uidvalidity(ctx, "INBOX")
         .await

src/constants.rs

@@ -227,6 +227,10 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
+// max. width/height of images
+pub const BALANCED_IMAGE_SIZE: u32 = 1280;
+pub const WORSE_IMAGE_SIZE: u32 = 640;
+
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 

src/context.rs

@@ -300,13 +300,11 @@ impl Context {
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
+            .get_config(Config::ConfiguredSentboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
+            .get_config(Config::ConfiguredMvboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
 
@@ -442,33 +440,19 @@ impl Context {
             .unwrap_or_default()
     }
 
-    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        folder_name.as_ref() == "INBOX"
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
-            .await;
-        if let Some(name) = sentbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
-            .await;
-
-        if let Some(name) = mvbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {

src/dc_receive_imf.rs

@@ -10,7 +10,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, Result};
+use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
@@ -227,6 +227,19 @@ pub async fn dc_receive_imf(
             context
                 .do_heuristics_moves(server_folder.as_ref(), insert_msg_id)
                 .await;
+            if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
+                // This is a Delta Chat MDN. Mark as read.
+                job::add(
+                    context,
+                    job::Job::new(
+                        Action::MarkseenMsgOnImap,
+                        insert_msg_id.to_u32(),
+                        Params::new(),
+                        0,
+                    ),
+                )
+                .await;
+            }
         }
     }
 
@@ -238,7 +251,7 @@ pub async fn dc_receive_imf(
     cleanup(context, &create_event_to_send, created_db_entries);
 
     mime_parser
-        .handle_reports(context, from_id, sent_timestamp)
+        .handle_reports(context, from_id, sent_timestamp, &mime_parser.parts)
         .await;
 
     Ok(())
@@ -313,8 +326,6 @@ async fn add_parts(
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
-    let mut sort_timestamp = 0;
-    let mut rcvd_timestamp = 0;
     let mut mime_in_reply_to = String::new();
     let mut mime_references = String::new();
     let mut incoming_origin = incoming_origin;
@@ -334,7 +345,7 @@ async fn add_parts(
         return Ok(());
     }
 
-    let mut msgrmsg = if mime_parser.has_chat_version() {
+    let mut is_dc_message = if mime_parser.has_chat_version() {
         MessengerMessage::Yes
     } else if is_reply_to_messenger_message(context, mime_parser).await {
         MessengerMessage::Reply
@@ -346,7 +357,7 @@ async fn add_parts(
     let show_emails =
         ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await).unwrap_or_default();
     if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage
-        && msgrmsg == MessengerMessage::No
+        && is_dc_message == MessengerMessage::No
     {
         // this message is a classic email not a chat-message nor a reply to one
         match show_emails {
@@ -375,7 +386,7 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get(HeaderDef::SecureJoin).is_some() {
-            msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting
+            is_dc_message = MessengerMessage::Yes; // avoid discarding by show_emails setting
             *chat_id = ChatId::new(0);
             allow_creation = true;
             match handle_securejoin_handshake(context, mime_parser, from_id).await {
@@ -407,6 +418,14 @@ async fn add_parts(
                 .await
                 .unwrap_or_default();
 
+        if chat_id.is_unset() && mime_parser.failure_report.is_some() {
+            *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+            info!(
+                context,
+                "Message belongs to an NDN and is not shown in a chat.",
+            );
+        }
+
         // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
         // it might also be blocked and displayed in the deaddrop as a result
         if chat_id.is_unset() {
@@ -500,7 +519,7 @@ async fn add_parts(
         if Blocked::Not != chat_id_blocked
             && state == MessageState::InFresh
             && !incoming_origin.is_known()
-            && msgrmsg == MessengerMessage::No
+            && is_dc_message == MessengerMessage::No
             && show_emails != ShowEmails::All
         {
             state = MessageState::InNoticed;
@@ -515,7 +534,7 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get(HeaderDef::SecureJoin).is_some() {
-            msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting
+            is_dc_message = MessengerMessage::Yes; // avoid discarding by show_emails setting
             *chat_id = ChatId::new(0);
             allow_creation = true;
             match observe_securejoin_on_other_device(context, mime_parser, to_id).await {
@@ -554,7 +573,7 @@ async fn add_parts(
                 }
             }
             if chat_id.is_unset() && allow_creation {
-                let create_blocked = if MessengerMessage::No != msgrmsg
+                let create_blocked = if MessengerMessage::No != is_dc_message
                     && !Contact::is_blocked_load(context, to_id).await
                 {
                     Blocked::Not
@@ -601,17 +620,9 @@ async fn add_parts(
     }
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
-    calc_timestamps(
-        context,
-        *chat_id,
-        from_id,
-        *sent_timestamp,
-        !seen,
-        &mut sort_timestamp,
-        sent_timestamp,
-        &mut rcvd_timestamp,
-    )
-    .await;
+    let rcvd_timestamp = time();
+    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, !seen).await;
+    *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
     chat_id.unarchive(context).await?;
@@ -647,7 +658,7 @@ async fn add_parts(
     let sent_timestamp = *sent_timestamp;
     let is_hidden = *hidden;
     let chat_id = *chat_id;
-    let is_mdn = !mime_parser.reports.is_empty();
+    let is_mdn = !mime_parser.mdn_reports.is_empty();
 
     // TODO: can this clone be avoided?
     let rfc724_mid = rfc724_mid.to_string();
@@ -664,8 +675,8 @@ async fn add_parts(
                     "INSERT INTO msgs \
          (rfc724_mid, server_folder, server_uid, chat_id, from_id, to_id, timestamp, \
          timestamp_sent, timestamp_rcvd, type, state, msgrmsg,  txt, txt_raw, param, \
-         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references) \
-         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?);",
+         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references, error) \
+         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?, ?);",
                 )?;
 
                 let is_location_kml = location_kml_is
@@ -699,7 +710,7 @@ async fn add_parts(
                     rcvd_timestamp,
                     part.typ,
                     state,
-                    msgrmsg,
+                    is_dc_message,
                     part.msg,
                     // txt_raw might contain invalid utf8
                     txt_raw,
@@ -709,6 +720,7 @@ async fn add_parts(
                     mime_headers,
                     mime_in_reply_to,
                     mime_references,
+                    part.error,
                 ])?;
 
                 drop(stmt);
@@ -749,6 +761,31 @@ async fn add_parts(
         }
     }
 
+    async fn update_last_subject(
+        context: &Context,
+        chat_id: ChatId,
+        mime_parser: &MimeMessage,
+    ) -> Result<()> {
+        let mut chat = Chat::load_from_db(context, chat_id).await?;
+        chat.param.set(
+            Param::LastSubject,
+            mime_parser
+                .get_subject()
+                .ok_or_else(|| format_err!("No subject in email"))?,
+        );
+        chat.update_param(context).await?;
+        Ok(())
+    }
+    update_last_subject(context, chat_id, mime_parser)
+        .await
+        .unwrap_or_else(|e| {
+            warn!(
+                context,
+                "Could not update LastSubject of chat: {}",
+                e.to_string()
+            )
+        });
+
     Ok(())
 }
 
@@ -812,41 +849,38 @@ async fn save_locations(
     }
 }
 
-#[allow(clippy::too_many_arguments)]
-async fn calc_timestamps(
+async fn calc_sort_timestamp(
     context: &Context,
-    chat_id: ChatId,
-    from_id: u32,
     message_timestamp: i64,
+    chat_id: ChatId,
     is_fresh_msg: bool,
-    sort_timestamp: &mut i64,
-    sent_timestamp: &mut i64,
-    rcvd_timestamp: &mut i64,
-) {
-    *rcvd_timestamp = time();
-    *sent_timestamp = message_timestamp;
-    if *sent_timestamp > *rcvd_timestamp {
-        *sent_timestamp = *rcvd_timestamp
-    }
-    *sort_timestamp = message_timestamp;
+) -> i64 {
+    let mut sort_timestamp = message_timestamp;
+
+    // get newest non fresh message for this chat
+    // update sort_timestamp if less than that
     if is_fresh_msg {
         let last_msg_time: Option<i64> = context
             .sql
             .query_get_value(
                 context,
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? and from_id!=? AND timestamp>=?",
-                paramsv![chat_id, from_id as i32, *sort_timestamp],
+                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
+                paramsv![chat_id, MessageState::InFresh],
             )
             .await;
+
         if let Some(last_msg_time) = last_msg_time {
-            if last_msg_time > 0 && *sort_timestamp <= last_msg_time {
-                *sort_timestamp = last_msg_time + 1;
+            if last_msg_time > sort_timestamp {
+                sort_timestamp = last_msg_time;
             }
         }
     }
-    if *sort_timestamp >= dc_smeared_time(context).await {
-        *sort_timestamp = dc_create_smeared_timestamp(context).await;
+
+    if sort_timestamp >= dc_smeared_time(context).await {
+        sort_timestamp = dc_create_smeared_timestamp(context).await;
     }
+
+    sort_timestamp
 }
 
 /// This function tries extracts the group-id from the message and returns the
@@ -889,32 +923,29 @@ async fn create_or_lookup_group(
     }
 
     if grpid.is_empty() {
-        if let Some(value) = mime_parser.get(HeaderDef::MessageId) {
-            if let Some(extracted_grpid) = dc_extract_grpid_from_rfc724_mid(&value) {
-                grpid = extracted_grpid.to_string();
-            }
-        }
-        if grpid.is_empty() {
-            if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
-                grpid = extracted_grpid.to_string();
-            } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References)
-            {
-                grpid = extracted_grpid.to_string();
-            } else {
-                return create_or_lookup_adhoc_group(
-                    context,
-                    mime_parser,
-                    allow_creation,
-                    create_blocked,
-                    from_id,
-                    to_ids,
-                )
-                .await
-                .map_err(|err| {
-                    info!(context, "could not create adhoc-group: {:?}", err);
-                    err
-                });
-            }
+        if let Some(extracted_grpid) = mime_parser
+            .get(HeaderDef::MessageId)
+            .and_then(|value| dc_extract_grpid_from_rfc724_mid(&value))
+        {
+            grpid = extracted_grpid.to_string();
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
+            grpid = extracted_grpid.to_string();
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) {
+            grpid = extracted_grpid.to_string();
+        } else {
+            return create_or_lookup_adhoc_group(
+                context,
+                mime_parser,
+                allow_creation,
+                create_blocked,
+                from_id,
+                to_ids,
+            )
+            .await
+            .map_err(|err| {
+                info!(context, "could not create adhoc-group: {:?}", err);
+                err
+            });
         }
     }
     // now we have a grpid that is non-empty
@@ -1001,7 +1032,7 @@ async fn create_or_lookup_group(
     let (mut chat_id, chat_id_verified, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
         .await
         .unwrap_or((ChatId::new(0), false, Blocked::Not));
-    if !chat_id.is_error() {
+    if !chat_id.is_unset() {
         if chat_id_verified {
             if let Err(err) =
                 check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
@@ -1032,7 +1063,7 @@ async fn create_or_lookup_group(
         .await
         .unwrap_or_default();
 
-    if chat_id.is_error()
+    if chat_id.is_unset()
             && !mime_parser.is_mailinglist_message()
             && !grpid.is_empty()
             && grpname.is_some()
@@ -1205,10 +1236,6 @@ async fn create_or_lookup_adhoc_group(
     from_id: u32,
     to_ids: &ContactIds,
 ) -> Result<(ChatId, Blocked)> {
-    // if we're here, no grpid was found, check if there is an existing
-    // ad-hoc group matching the to-list or if we should and can create one
-    // (we do not want to heuristically look at the likely mangled Subject)
-
     if mime_parser.is_mailinglist_message() {
         // XXX we could parse List-* headers and actually create and
         // manage a mailing list group, eventually
@@ -1219,6 +1246,10 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
+    // if we're here, no grpid was found, check if there is an existing
+    // ad-hoc group matching the to-list or if we should and can create one
+    // (we do not want to heuristically look at the likely mangled Subject)
+
     let mut member_ids: Vec<u32> = to_ids.iter().copied().collect();
     if !member_ids.contains(&from_id) {
         member_ids.push(from_id);
@@ -1271,6 +1302,24 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
+    if mime_parser.decrypting_failed {
+        // Do not create a new ad-hoc group if the message cannot be
+        // decrypted.
+        //
+        // The subject may be encrypted and contain a placeholder such
+        // as "...". Besides that, it is possible that the message was
+        // sent to a valid, yet unknown group, which was rejected
+        // because Chat-Group-Name, which is in the encrypted part,
+        // was not found. Generating a new ID in this case would
+        // result in creation of a twin group with a different group
+        // ID.
+        warn!(
+            context,
+            "not creating ad-hoc group for message that cannot be decrypted"
+        );
+        return Ok((ChatId::new(0), Blocked::Not));
+    }
+
     // we do not check if the message is a reply to another group, this may result in
     // chats with unclear member list. instead we create a new group in the following lines ...
 
@@ -1740,7 +1789,7 @@ mod tests {
     use crate::chat::ChatVisibility;
     use crate::chatlist::Chatlist;
     use crate::message::Message;
-    use crate::test_utils::{dummy_context, TestContext};
+    use crate::test_utils::*;
 
     #[test]
     fn test_hex_hash() {
@@ -1836,23 +1885,6 @@ mod tests {
         assert!(!is_msgrmsg_rfc724_mid(&t.ctx, "nonexistant@message.id").await);
     }
 
-    async fn configured_offline_context() -> TestContext {
-        let t = dummy_context().await;
-        t.ctx
-            .set_config(Config::Addr, Some("alice@example.org"))
-            .await
-            .unwrap();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.org"))
-            .await
-            .unwrap();
-        t.ctx
-            .set_config(Config::Configured, Some("1"))
-            .await
-            .unwrap();
-        t
-    }
-
     static MSGRMSG: &[u8] = b"From: Bob <bob@example.org>\n\
                     To: alice@example.org\n\
                     Chat-Version: 1.0\n\
@@ -2314,4 +2346,182 @@ mod tests {
             "Carl"
         );
     }
+
+    #[async_std::test]
+    async fn test_parse_ndn_tiscali() {
+        test_parse_ndn(
+            "alice@tiscali.it",
+            "shenauithz@testrun.org",
+            "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
+            include_bytes!("../test-data/message/tiscali_ndn.eml"),
+            "",
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_testrun() {
+        test_parse_ndn(
+            "alice@testrun.org",
+            "hcksocnsofoejx@five.chat",
+            "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
+            include_bytes!("../test-data/message/testrun_ndn.eml"),
+            "Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_yahoo() {
+        test_parse_ndn(
+            "alice@yahoo.com",
+            "haeclirth.sinoenrat@yahoo.com",
+            "1680295672.3657931.1591783872936@mail.yahoo.com",
+            include_bytes!("../test-data/message/yahoo_ndn.eml"),
+            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_gmail() {
+        test_parse_ndn(
+            "alice@gmail.com",
+            "assidhfaaspocwaeofi@gmail.com",
+            "CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com",
+            include_bytes!("../test-data/message/gmail_ndn.eml"),
+            "Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp",
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_gmx() {
+        test_parse_ndn(
+            "alice@gmx.com",
+            "snaerituhaeirns@gmail.com",
+            "9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de",
+            include_bytes!("../test-data/message/gmx_ndn.eml"),
+            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_posteo() {
+        test_parse_ndn(
+            "alice@posteo.org",
+            "hanerthaertidiuea@gmx.de",
+            "04422840-f884-3e37-5778-8192fe22d8e1@posteo.de",
+            include_bytes!("../test-data/message/posteo_ndn.eml"),
+            "Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)",
+        )
+        .await;
+    }
+
+    // ndn = Non Delivery Notification
+    async fn test_parse_ndn(
+        self_addr: &str,
+        foreign_addr: &str,
+        rfc724_mid_outgoing: &str,
+        raw_ndn: &[u8],
+        error_msg: &str,
+    ) {
+        let t = configured_offline_context_with_addr(self_addr).await;
+
+        dc_receive_imf(
+            &t.ctx,
+            format!(
+                "From: {}\n\
+                To: {}\n\
+                Subject: foo\n\
+                Message-ID: <{}>\n\
+                Chat-Version: 1.0\n\
+                Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                \n\
+                hello\n",
+                self_addr, foreign_addr, rfc724_mid_outgoing
+            )
+            .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+
+        // Check that the ndn would be downloaded:
+        let headers = mailparse::parse_mail(raw_ndn).unwrap().headers;
+        assert!(
+            crate::imap::prefetch_should_download(&t.ctx, &headers, ShowEmails::Off)
+                .await
+                .unwrap()
+        );
+
+        dc_receive_imf(&t.ctx, raw_ndn, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+
+        assert_eq!(msg.state, MessageState::OutFailed);
+        assert_eq!(msg.error, error_msg);
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_group_msg() {
+        let t = configured_offline_context_with_addr("alice@gmail.com").await;
+
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@gmail.com\n\
+                 To: bob@example.org, assidhfaaspocwaeofi@gmail.com\n\
+                 Subject: foo\n\
+                 Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde\n\
+                 Chat-Group-Name: foo\n\
+                 Chat-Disposition-Notification-To: alice@example.org\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+
+        let raw = include_bytes!("../test-data/message/gmail_ndn_group.eml");
+        dc_receive_imf(&t.ctx, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+
+        assert_eq!(msg.state, MessageState::OutFailed);
+
+        let msgs = chat::get_chat_msgs(&t.ctx, msg.chat_id, 0, None).await;
+        let last_msg = Message::load_from_db(&t.ctx, *msgs.last().unwrap())
+            .await
+            .unwrap();
+
+        assert_eq!(
+            last_msg.text,
+            Some(
+                t.ctx
+                    .stock_string_repl_str(
+                        StockMessage::FailedSendingTo,
+                        "assidhfaaspocwaeofi@gmail.com",
+                    )
+                    .await,
+            )
+        );
+        assert_eq!(last_msg.from_id, DC_CONTACT_ID_INFO);
+    }
 }

src/e2ee.rs

@@ -11,7 +11,7 @@ use crate::context::Context;
 use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, SignedPublicKey, SignedSecretKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -119,7 +119,7 @@ pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
@@ -212,7 +212,7 @@ async fn decrypt_if_autocrypt_message<'a>(
     mail: &ParsedMail<'a>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<String>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     //  The returned bool is true if we detected an Autocrypt-encrypted
     // message and successfully decrypted it. Decryption then modifies the
@@ -244,7 +244,7 @@ async fn decrypt_part(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<String>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 

src/headerdef.rs

@@ -21,6 +21,7 @@ pub enum HeaderDef {
     References,
     InReplyTo,
     Precedence,
+    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,

src/imap/client.rs

@@ -7,7 +7,7 @@ use async_imap::{
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
+use crate::login_param::dc_build_tls;
 
 use super::session::SessionStream;
 
@@ -78,10 +78,10 @@ impl Client {
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        certificate_checks: CertificateChecks,
+        strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
+        let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
@@ -118,16 +118,12 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
+    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {
             let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(certificate_checks);
+            let tls = dc_build_tls(strict_tls);
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();

src/imap/idle.rs

@@ -4,7 +4,7 @@ use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::context::Context;
+use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::select_folder;
 use super::session::Session;
@@ -34,7 +34,11 @@ impl Imap {
         self.config.can_idle
     }
 
-    pub async fn idle(&mut self, context: &Context, watch_folder: Option<String>) -> Result<bool> {
+    pub async fn idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> Result<InterruptInfo> {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
@@ -46,7 +50,7 @@ impl Imap {
 
         let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
-        let mut probe_network = false;
+        let mut info = Default::default();
 
         if let Some(session) = session {
             let mut handle = session.idle();
@@ -58,7 +62,7 @@ impl Imap {
 
             enum Event {
                 IdleResponse(IdleResponse),
-                Interrupt(bool),
+                Interrupt(InterruptInfo),
             }
 
             if self.skip_next_idle_wait {
@@ -90,8 +94,8 @@ impl Imap {
                     Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
                         info!(context, "Idle wait was interrupted");
                     }
-                    Ok(Event::Interrupt(probe)) => {
-                        probe_network = probe;
+                    Ok(Event::Interrupt(i)) => {
+                        info = i;
                         info!(context, "Idle wait was interrupted");
                     }
                     Err(err) => {
@@ -125,14 +129,14 @@ impl Imap {
             }
         }
 
-        Ok(probe_network)
+        Ok(info)
     }
 
     pub(crate) async fn fake_idle(
         &mut self,
         context: &Context,
         watch_folder: Option<String>,
-    ) -> bool {
+    ) -> InterruptInfo {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
@@ -144,7 +148,7 @@ impl Imap {
             return self.idle_interrupt.recv().await.unwrap_or_default();
         }
 
-        let mut probe_network = false;
+        let mut info: InterruptInfo = Default::default();
         if self.skip_next_idle_wait {
             // interrupt_idle has happened before we
             // provided self.interrupt
@@ -157,10 +161,10 @@ impl Imap {
 
             enum Event {
                 Tick,
-                Interrupt(bool),
+                Interrupt(InterruptInfo),
             }
             // loop until we are interrupted or if we fetched something
-            probe_network =
+            info =
                 loop {
                     use futures::future::FutureExt;
                     match interval
@@ -181,7 +185,7 @@ impl Imap {
                             }
                             if self.config.can_idle {
                                 // we only fake-idled because network was gone during IDLE, probably
-                                break false;
+                                break InterruptInfo::new(false, None);
                             }
                             info!(context, "fake_idle is connected");
                             // we are connected, let's see if fetching messages results
@@ -194,7 +198,7 @@ impl Imap {
                                     Ok(res) => {
                                         info!(context, "fetch_new_messages returned {:?}", res);
                                         if res {
-                                            break false;
+                                            break InterruptInfo::new(false, None);
                                         }
                                     }
                                     Err(err) => {
@@ -204,9 +208,9 @@ impl Imap {
                                 }
                             }
                         }
-                        Event::Interrupt(probe_network) => {
+                        Event::Interrupt(info) => {
                             // Interrupt
-                            break probe_network;
+                            break info;
                         }
                     }
                 };
@@ -222,6 +226,6 @@ impl Imap {
                 / 1000.,
         );
 
-        probe_network
+        info
     }
 }

src/imap/mod.rs

@@ -27,7 +27,8 @@ use crate::message::{self, update_server_uid};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
-use crate::stock::StockMessage;
+use crate::provider::get_provider_info;
+use crate::{scheduler::InterruptInfo, stock::StockMessage};
 
 mod client;
 mod idle;
@@ -109,7 +110,7 @@ const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
-    idle_interrupt: Receiver<bool>,
+    idle_interrupt: Receiver<InterruptInfo>,
     config: ImapConfig,
     session: Option<Session>,
     connected: bool,
@@ -149,7 +150,7 @@ struct ImapConfig {
     pub imap_port: u16,
     pub imap_user: String,
     pub imap_pw: String,
-    pub certificate_checks: CertificateChecks,
+    pub strict_tls: bool,
     pub server_flags: usize,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
@@ -169,7 +170,7 @@ impl Default for ImapConfig {
             imap_port: 0,
             imap_user: "".into(),
             imap_pw: "".into(),
-            certificate_checks: Default::default(),
+            strict_tls: false,
             server_flags: 0,
             selected_folder: None,
             selected_mailbox: None,
@@ -181,7 +182,7 @@ impl Default for ImapConfig {
 }
 
 impl Imap {
-    pub fn new(idle_interrupt: Receiver<bool>) -> Self {
+    pub fn new(idle_interrupt: Receiver<InterruptInfo>) -> Self {
         Imap {
             idle_interrupt,
             config: Default::default(),
@@ -228,7 +229,7 @@ impl Imap {
                 match Client::connect_insecure((imap_server, imap_port)).await {
                     Ok(client) => {
                         if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.certificate_checks).await
+                            client.secure(imap_server, config.strict_tls).await
                         } else {
                             Ok(client)
                         }
@@ -240,12 +241,8 @@ impl Imap {
                 let imap_server: &str = config.imap_server.as_ref();
                 let imap_port = config.imap_port;
 
-                Client::connect_secure(
-                    (imap_server, imap_port),
-                    imap_server,
-                    config.certificate_checks,
-                )
-                .await
+                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
+                    .await
             };
 
         let login_res = match connection_res {
@@ -295,6 +292,8 @@ impl Imap {
 
         match login_res {
             Ok(session) => {
+                // needs to be set here to ensure it is set on reconnects.
+                self.connected = true;
                 self.session = Some(session);
                 Ok(())
             }
@@ -315,9 +314,15 @@ impl Imap {
     }
 
     async fn unsetup_handle(&mut self, context: &Context) {
+        // Close folder if messages should be expunged
+        if let Err(err) = self.close_folder(context).await {
+            warn!(context, "failed to close folder: {:?}", err);
+        }
+
+        // Logout from the server
         if let Some(mut session) = self.session.take() {
-            if let Err(err) = session.close().await {
-                warn!(context, "failed to close connection: {:?}", err);
+            if let Err(err) = session.logout().await {
+                warn!(context, "failed to logout: {:?}", err);
             }
         }
         self.connected = false;
@@ -377,7 +382,15 @@ impl Imap {
             config.imap_port = imap_port;
             config.imap_user = imap_user.to_string();
             config.imap_pw = imap_pw.to_string();
-            config.certificate_checks = lp.imap_certificate_checks;
+            let provider = get_provider_info(&lp.addr);
+            config.strict_tls = match lp.imap_certificate_checks {
+                CertificateChecks::Automatic => {
+                    provider.map_or(false, |provider| provider.strict_tls)
+                }
+                CertificateChecks::Strict => true,
+                CertificateChecks::AcceptInvalidCertificates
+                | CertificateChecks::AcceptInvalidCertificates2 => false,
+            };
             config.server_flags = server_flags;
         }
 
@@ -798,7 +811,6 @@ impl Imap {
                     Ok(_) => Some(server_uid),
                     Err(err) => {
                         warn!(context, "dc_receive_imf error: {}", err);
-                        read_errors += 1;
                         None
                     }
                 }
@@ -972,7 +984,7 @@ impl Imap {
         uid: u32,
     ) -> Option<ImapActionResult> {
         if uid == 0 {
-            return Some(ImapActionResult::Failed);
+            return Some(ImapActionResult::RetryLater);
         }
         if !self.is_connected() {
             // currently jobs are only performed on the INBOX thread
@@ -1140,54 +1152,54 @@ impl Imap {
                 }
             };
 
+            let mut delimiter = ".".to_string();
+            let mut delimiter_is_default = true;
             let mut sentbox_folder = None;
             let mut mvbox_folder = None;
-
-            let mut delimiter = ".".to_string();
-            if let Some(folder) = folders.next().await {
-                let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
-                if let Some(d) = folder.delimiter() {
-                    if !d.is_empty() {
-                        delimiter = d.to_string();
-                    }
-                }
-            }
-            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
-            let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
+            let mut fallback_folder = get_fallback_folder(&delimiter);
 
             while let Some(folder) = folders.next().await {
                 let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
                 info!(context, "Scanning folder: {:?}", folder);
 
-                if mvbox_folder.is_none()
-                    && (folder.name() == "DeltaChat" || folder.name() == fallback_folder)
-                {
-                    mvbox_folder = Some(folder.name().to_string());
-                }
-
-                if sentbox_folder.is_none() {
-                    if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
-                        sentbox_folder = Some(folder);
-                    } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
-                        sentbox_folder = Some(folder);
+                // Update the delimiter iff there is a different one, but only once.
+                if let Some(d) = folder.delimiter() {
+                    if delimiter_is_default && !d.is_empty() && delimiter != d {
+                        delimiter = d.to_string();
+                        fallback_folder = get_fallback_folder(&delimiter);
+                        delimiter_is_default = false;
                     }
                 }
 
-                if mvbox_folder.is_some() && sentbox_folder.is_some() {
-                    break;
+                if folder.name() == "DeltaChat" {
+                    // Always takes precendent
+                    mvbox_folder = Some(folder.name().to_string());
+                } else if folder.name() == fallback_folder {
+                    // only set iff none has been already set
+                    if mvbox_folder.is_none() {
+                        mvbox_folder = Some(folder.name().to_string());
+                    }
+                } else if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
+                    // Always takes precedent
+                    sentbox_folder = Some(folder.name().to_string());
+                } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
+                    // only set iff none has been already set
+                    if sentbox_folder.is_none() {
+                        sentbox_folder = Some(folder.name().to_string());
+                    }
                 }
             }
-            info!(context, "sentbox folder is {:?}", sentbox_folder);
-
             drop(folders);
 
+            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
+            info!(context, "sentbox folder is {:?}", sentbox_folder);
+
             if mvbox_folder.is_none() && create_mvbox {
                 info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
 
                 match session.create("DeltaChat").await {
                     Ok(_) => {
                         mvbox_folder = Some("DeltaChat".into());
-
                         info!(context, "MVBOX-folder created.",);
                     }
                     Err(err) => {
@@ -1221,23 +1233,16 @@ impl Imap {
                 }
             }
             context
-                .sql
-                .set_raw_config(context, "configured_inbox_folder", Some("INBOX"))
+                .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
                 .await?;
             if let Some(ref mvbox_folder) = mvbox_folder {
                 context
-                    .sql
-                    .set_raw_config(context, "configured_mvbox_folder", Some(mvbox_folder))
+                    .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
                     .await?;
             }
             if let Some(ref sentbox_folder) = sentbox_folder {
                 context
-                    .sql
-                    .set_raw_config(
-                        context,
-                        "configured_sentbox_folder",
-                        Some(sentbox_folder.name()),
-                    )
+                    .set_config(Config::ConfiguredSentboxFolder, Some(sentbox_folder))
                     .await?;
             }
             context
@@ -1395,7 +1400,11 @@ async fn precheck_imf(
         }
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
-            update_server_uid(context, &rfc724_mid, server_folder, server_uid).await;
+            update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
+            context
+                .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
+                .await;
+            info!(context, "Updating server_uid and interrupting")
         }
         Ok(true)
     } else {
@@ -1439,7 +1448,7 @@ async fn prefetch_is_reply_to_chat_message(
     false
 }
 
-async fn prefetch_should_download(
+pub(crate) async fn prefetch_should_download(
     context: &Context,
     headers: &[mailparse::MailHeader<'_>],
     show_emails: ShowEmails,
@@ -1447,6 +1456,13 @@ async fn prefetch_should_download(
     let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
     let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await;
 
+    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
+        let from = from.to_ascii_lowercase();
+        from.contains("mailer-daemon") || from.contains("mail-daemon")
+    } else {
+        false
+    };
+
     // Autocrypt Setup Message should be shown even if it is from non-chat client.
     let is_autocrypt_setup_message = headers
         .get_header_value(HeaderDef::AutocryptSetupMessage)
@@ -1457,6 +1473,7 @@ async fn prefetch_should_download(
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message
+        || maybe_ndn
         || match show_emails {
             ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
             ShowEmails::AcceptedContacts => {
@@ -1514,3 +1531,7 @@ async fn message_needs_processing(
 
     true
 }
+
+fn get_fallback_folder(delimiter: &str) -> String {
+    format!("INBOX{}DeltaChat", delimiter)
+}

src/imap/select_folder.rs

@@ -27,7 +27,7 @@ impl Imap {
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 

src/job.rs

@@ -31,7 +31,7 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
-use crate::sql;
+use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
@@ -190,7 +190,7 @@ impl Job {
     /// Saves the job to the database, creating a new entry if necessary.
     ///
     /// The Job is consumed by this method.
-    pub async fn save(self, context: &Context) -> Result<()> {
+    pub(crate) async fn save(self, context: &Context) -> Result<()> {
         let thread: Thread = self.action.into();
 
         info!(context, "saving job for {}-thread: {:?}", thread, self);
@@ -329,7 +329,7 @@ impl Job {
         }
     }
 
-    pub async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
         if !smtp.is_connected().await {
             let loginparam = LoginParam::from_database(context, "configured_").await;
@@ -498,16 +498,13 @@ impl Job {
     }
 
     async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-
-        if let Err(err) = imap.ensure_configured_folders(context, true).await {
-            warn!(context, "could not configure folders: {:?}", err);
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
-        let dest_folder = context
-            .sql
-            .get_raw_config(context, "configured_mvbox_folder")
-            .await;
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
 
         if let Some(dest_folder) = dest_folder {
             let server_folder = msg.server_folder.as_ref().unwrap();
@@ -518,7 +515,7 @@ impl Job {
             {
                 ImapActionResult::RetryLater => Status::RetryLater,
                 ImapActionResult::Success => {
-                    // XXX Rust-Imap provides no target uid on mv, so just set it to 0
+                    // Rust-Imap provides no target uid on mv, so just set it to 0, update again when precheck_imf() is called for the moved message
                     message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await;
                     Status::Finished(Ok(()))
                 }
@@ -541,6 +538,11 @@ impl Job {
     /// records pointing to the same message on the server, the job
     /// also removes the message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         if !msg.rfc724_mid.is_empty() {
@@ -611,12 +613,13 @@ impl Job {
     }
 
     async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = context
-                .sql
-                .get_raw_config(context, "configured_mvbox_folder")
-                .await
-            {
+            if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
                 imap.empty_folder(context, &mvbox_folder).await;
             }
         }
@@ -627,6 +630,11 @@ impl Job {
     }
 
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
@@ -798,7 +806,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
 
     if rendered_msg.is_encrypted && !needs_encryption {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
-        msg.save_param_to_disk(context).await;
+        msg.update_param(context).await;
     }
 
     ensure!(!recipients.is_empty(), "no recipients for smtp job set");
@@ -815,8 +823,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     Ok(Some(job))
 }
 
-#[derive(Debug)]
-pub enum Connection<'a> {
+pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
@@ -1029,14 +1036,18 @@ pub async fn add(context: &Context, job: Job) {
             | Action::MarkseenMsgOnImap
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
-                context.interrupt_inbox(false).await;
+                context
+                    .interrupt_inbox(InterruptInfo::new(false, None))
+                    .await;
             }
             Action::MaybeSendLocations
             | Action::MaybeSendLocationsEnded
             | Action::SendMdn
             | Action::SendMsgToSmtp => {
                 info!(context, "interrupt: smtp");
-                context.interrupt_smtp(false).await;
+                context
+                    .interrupt_smtp(InterruptInfo::new(false, None))
+                    .await;
             }
         }
     }
@@ -1051,38 +1062,49 @@ pub async fn add(context: &Context, job: Job) {
 pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
-    probe_network: bool,
+    info: &InterruptInfo,
 ) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
-    let query = if !probe_network {
+
+    let query;
+    let params;
+    let t = time();
+    let m;
+    let thread_i = thread as i64;
+
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
+SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+FROM jobs
+WHERE thread=? AND foreign_id=?
+ORDER BY action DESC, added_timestamp
+LIMIT 1;
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
         // processing for first-try and after backoff-timeouts:
         // process jobs in the order they were added.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND desired_timestamp<=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#
+"#;
+        params = paramsv![thread_i, t];
     } else {
         // processing after call to dc_maybe_network():
         // process _all_ pending jobs that failed before
         // in the order of their backoff-times.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND tries>0
 ORDER BY desired_timestamp, action DESC
 LIMIT 1;
-"#
-    };
-
-    let thread_i = thread as i64;
-    let t = time();
-    let params = if !probe_network {
-        paramsv![thread_i, t]
-    } else {
-        paramsv![thread_i]
+"#;
+        params = paramsv![thread_i];
     };
 
     let job = loop {
@@ -1189,11 +1211,21 @@ mod tests {
         // all jobs.
         let t = dummy_context().await;
         insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_none());
 
         insert_job(&t.ctx, 1).await;
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_some());
     }
 
@@ -1203,7 +1235,12 @@ mod tests {
 
         insert_job(&t.ctx, 1).await;
 
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_some());
     }
 }

src/key.rs

@@ -9,6 +9,7 @@ use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::*;
@@ -106,7 +107,7 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
 
     /// The fingerprint for the key.
     fn fingerprint(&self) -> Fingerprint {
-        Fingerprint::new(KeyTrait::fingerprint(self))
+        Fingerprint::new(KeyTrait::fingerprint(self)).expect("Invalid fingerprint from rpgp")
     }
 }
 
@@ -354,12 +355,15 @@ pub async fn store_self_keypair(
 }
 
 /// A key fingerprint
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
-    pub fn new(v: Vec<u8>) -> Fingerprint {
-        Fingerprint(v)
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+        match v.len() {
+            20 => Ok(Fingerprint(v)),
+            _ => Err(FingerprintError::WrongLength),
+        }
     }
 
     /// Make a hex string from the fingerprint.
@@ -389,42 +393,26 @@ impl fmt::Display for Fingerprint {
 
 /// Parse a human-readable or otherwise formatted fingerprint.
 impl std::str::FromStr for Fingerprint {
-    type Err = hex::FromHexError;
+    type Err = FingerprintError;
 
     fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
         let hex_repr: String = input
+            .to_uppercase()
             .chars()
             .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
             .collect();
         let v: Vec<u8> = hex::decode(hex_repr)?;
-        Ok(Fingerprint(v))
+        let fp = Fingerprint::new(v)?;
+        Ok(fp)
     }
 }
 
-/// Bring a human-readable or otherwise formatted fingerprint back to the 40-characters-uppercase-hex format.
-pub fn dc_normalize_fingerprint(fp: &str) -> String {
-    fp.to_uppercase()
-        .chars()
-        .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
-        .collect()
-}
-
-/// Make a fingerprint human-readable, in hex format.
-pub fn dc_format_fingerprint(fingerprint: &str) -> String {
-    // split key into chunks of 4 with space, and 20 newline
-    let mut res = String::new();
-
-    for (i, c) in fingerprint.chars().enumerate() {
-        if i > 0 && i % 20 == 0 {
-            res += "\n";
-        } else if i > 0 && i % 4 == 0 {
-            res += " ";
-        }
-
-        res += &c.to_string();
-    }
-
-    res
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
 }
 
 #[cfg(test)]
@@ -432,6 +420,8 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
 
+    use std::error::Error;
+
     use async_std::sync::Arc;
     use lazy_static::lazy_static;
 
@@ -439,23 +429,6 @@ mod tests {
         static ref KEYPAIR: KeyPair = alice_keypair();
     }
 
-    #[test]
-    fn test_normalize_fingerprint() {
-        let fingerprint = dc_normalize_fingerprint(" 1234  567890 \n AbcD abcdef ABCDEF ");
-
-        assert_eq!(fingerprint, "1234567890ABCDABCDEFABCDEF");
-    }
-
-    #[test]
-    fn test_format_fingerprint() {
-        let fingerprint = dc_format_fingerprint("1234567890ABCDABCDEFABCDEF1234567890ABCD");
-
-        assert_eq!(
-            fingerprint,
-            "1234 5678 90AB CDAB CDEF\nABCD EF12 3456 7890 ABCD"
-        );
-    }
-
     #[test]
     fn test_from_armored_string() {
         let (private_key, _) = SignedSecretKey::from_asc(
@@ -586,7 +559,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_public() {
         let t = dummy_context().await;
         t.ctx
@@ -598,7 +570,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_secret() {
         let t = dummy_context().await;
         t.ctx
@@ -610,7 +581,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
@@ -685,32 +655,46 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 
     #[test]
     fn test_fingerprint_from_str() {
-        let res = Fingerprint::new(vec![1, 2, 4, 8, 16, 32, 64, 128, 255]);
+        let res = Fingerprint::new(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
 
-        let fp: Fingerprint = "0102040810204080FF".parse().unwrap();
+        let fp: Fingerprint = "0102030405060708090A0B0c0d0e0F1011121314".parse().unwrap();
         assert_eq!(fp, res);
 
-        let fp: Fingerprint = "zzzz 0102 0408\n1020 4080 FF zzz".parse().unwrap();
+        let fp: Fingerprint = "zzzz 0102 0304 0506\n0708090a0b0c0D0E0F1011121314 yyy"
+            .parse()
+            .unwrap();
         assert_eq!(fp, res);
 
         let err = "1".parse::<Fingerprint>().err().unwrap();
-        assert_eq!(err, hex::FromHexError::OddLength);
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
     }
 
     #[test]
     fn test_fingerprint_hex() {
-        let fp = Fingerprint::new(vec![1, 2, 4, 8, 16, 32, 64, 128, 255]);
-        assert_eq!(fp.hex(), "0102040810204080FF");
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(fp.hex(), "0102040810204080FF0A0B0C0D0E0F1011121314");
     }
 
     #[test]
     fn test_fingerprint_to_string() {
         let fp = Fingerprint::new(vec![
-            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255,
-        ]);
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255, 19, 20,
+        ])
+        .unwrap();
         assert_eq!(
             fp.to_string(),
-            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF"
+            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF 1314"
         );
     }
 }

src/lib.rs

@@ -11,8 +11,6 @@ extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 

src/location.rs

@@ -530,7 +530,7 @@ pub async fn save(
             accuracy,
             ..
         } = location;
-        context
+        let (loc_id, ts) = context
             .sql
             .with_conn(move |mut conn| {
                 let mut stmt_test = conn
@@ -569,9 +569,11 @@ pub async fn save(
                         )?;
                     }
                 }
-                Ok(())
+                Ok((newest_location_id, newest_timestamp))
             })
             .await?;
+        newest_timestamp = ts;
+        newest_location_id = loc_id;
     }
 
     Ok(newest_location_id)

src/login_param.rs

@@ -130,10 +130,6 @@ impl LoginParam {
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
     pub async fn save_to_database(
         &self,
@@ -277,21 +273,15 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-    match certificate_checks {
-        CertificateChecks::Automatic => {
-            // Same as AcceptInvalidCertificates for now.
-            // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
-        }
-        CertificateChecks::Strict => tls_builder,
-        CertificateChecks::AcceptInvalidCertificates
-        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
+
+    if strict_tls {
+        tls_builder
+    } else {
+        tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
+            .danger_accept_invalid_certs(true)
     }
 }
 

src/lot.rs

@@ -1,5 +1,7 @@
 use deltachat_derive::{FromSql, ToSql};
 
+use crate::key::Fingerprint;
+
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
 /// Lot objects are created
@@ -14,7 +16,7 @@ pub struct Lot {
     pub(crate) timestamp: i64,
     pub(crate) state: LotState,
     pub(crate) id: u32,
-    pub(crate) fingerprint: Option<String>,
+    pub(crate) fingerprint: Option<Fingerprint>,
     pub(crate) invitenumber: Option<String>,
     pub(crate) auth: Option<String>,
 }

src/message.rs

@@ -14,7 +14,7 @@ use crate::error::{ensure, Error};
 use crate::events::Event;
 use crate::job::{self, Action};
 use crate::lot::{Lot, LotState, Meaning};
-use crate::mimeparser::SystemMessage;
+use crate::mimeparser::{FailureReport, SystemMessage};
 use crate::param::*;
 use crate::pgp::*;
 use crate::stock::StockMessage;
@@ -255,6 +255,7 @@ pub struct Message {
     pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
+    pub(crate) error: String,
     pub(crate) param: Params,
 }
 
@@ -289,6 +290,7 @@ impl Message {
                     "    m.timestamp_rcvd AS timestamp_rcvd,",
                     "    m.type AS type,",
                     "    m.state AS state,",
+                    "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.txt AS txt,",
                     "    m.param AS param,",
@@ -316,6 +318,7 @@ impl Message {
                     msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
+                    msg.error = row.get("error")?;
                     msg.is_dc_message = row.get("msgrmsg")?;
 
                     let text;
@@ -390,7 +393,7 @@ impl Message {
                     }
 
                     if !self.id.is_unset() {
-                        self.save_param_to_disk(context).await;
+                        self.update_param(context).await;
                     }
                 }
             }
@@ -643,10 +646,10 @@ impl Message {
         if duration > 0 {
             self.param.set_int(Param::Duration, duration);
         }
-        self.save_param_to_disk(context).await;
+        self.update_param(context).await;
     }
 
-    pub async fn save_param_to_disk(&mut self, context: &Context) -> bool {
+    pub async fn update_param(&mut self, context: &Context) -> bool {
         context
             .sql
             .execute(
@@ -763,9 +766,10 @@ impl From<MessageState> for LotState {
 impl MessageState {
     pub fn can_fail(self) -> bool {
         match self {
-            MessageState::OutPreparing | MessageState::OutPending | MessageState::OutDelivered => {
-                true
-            }
+            MessageState::OutPreparing
+            | MessageState::OutPending
+            | MessageState::OutDelivered
+            | MessageState::OutMdnRcvd => true, // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
             _ => false,
         }
     }
@@ -937,8 +941,9 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
     }
 
     ret += "\n";
-    if let Some(err) = msg.param.get(Param::Error) {
-        ret += &format!("Error: {}", err)
+
+    if !msg.error.is_empty() {
+        ret += &format!("Error: {}", msg.error);
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -1251,33 +1256,38 @@ pub async fn exists(context: &Context, msg_id: MsgId) -> bool {
 
 pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id).await {
+        let error = error.map(|e| e.as_ref().to_string()).unwrap_or_default();
         if msg.state.can_fail() {
             msg.state = MessageState::OutFailed;
-        }
-        if let Some(error) = error {
-            msg.param.set(Param::Error, error.as_ref());
-            warn!(context, "Message failed: {}", error.as_ref());
+            warn!(context, "{} failed: {}", msg_id, error);
+        } else {
+            warn!(
+                context,
+                "{} seems to have failed ({}), but state is {}", msg_id, error, msg.state
+            )
         }
 
-        if context
+        match context
             .sql
             .execute(
-                "UPDATE msgs SET state=?, param=? WHERE id=?;",
-                paramsv![msg.state, msg.param.to_string(), msg_id],
+                "UPDATE msgs SET state=?, error=? WHERE id=?;",
+                paramsv![msg.state, error, msg_id],
             )
             .await
-            .is_ok()
         {
-            context.emit_event(Event::MsgFailed {
+            Ok(_) => context.emit_event(Event::MsgFailed {
                 chat_id: msg.chat_id,
                 msg_id,
-            });
+            }),
+            Err(e) => {
+                warn!(context, "{:?}", e);
+            }
         }
     }
 }
 
 /// returns Some if an event should be send
-pub async fn mdn_from_ext(
+pub async fn handle_mdn(
     context: &Context,
     from_id: u32,
     rfc724_mid: &str,
@@ -1318,10 +1328,10 @@ pub async fn mdn_from_ext(
     if let Ok((msg_id, chat_id, chat_type, msg_state)) = res {
         let mut read_by_all = false;
 
-        // if already marked as MDNS_RCVD msgstate_can_fail() returns false.
-        // however, it is important, that ret_msg_id is set above as this
-        // will allow the caller eg. to move the message away
-        if msg_state.can_fail() {
+        if msg_state == MessageState::OutPreparing
+            || msg_state == MessageState::OutPending
+            || msg_state == MessageState::OutDelivered
+        {
             let mdn_already_in_table = context
                 .sql
                 .exists(
@@ -1384,6 +1394,69 @@ pub async fn mdn_from_ext(
     None
 }
 
+/// Marks a message as failed after an ndn (non-delivery-notification) arrived.
+/// Where appropriate, also adds an info message telling the user which of the recipients of a group message failed.
+pub(crate) async fn handle_ndn(
+    context: &Context,
+    failed: &FailureReport,
+    error: Option<impl AsRef<str>>,
+) {
+    if failed.rfc724_mid.is_empty() {
+        return;
+    }
+
+    let res = context
+        .sql
+        .query_row(
+            concat!(
+                "SELECT",
+                "    m.id AS msg_id,",
+                "    c.id AS chat_id,",
+                "    c.type AS type",
+                " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+                " WHERE rfc724_mid=? AND from_id=1",
+            ),
+            paramsv![failed.rfc724_mid],
+            |row| {
+                Ok((
+                    row.get::<_, MsgId>("msg_id")?,
+                    row.get::<_, ChatId>("chat_id")?,
+                    row.get::<_, Chattype>("type")?,
+                ))
+            },
+        )
+        .await;
+    if let Err(ref err) = res {
+        info!(context, "Failed to select NDN {:?}", err);
+    }
+
+    if let Ok((msg_id, chat_id, chat_type)) = res {
+        set_msg_failed(context, msg_id, error).await;
+
+        if chat_type == Chattype::Group || chat_type == Chattype::VerifiedGroup {
+            if let Some(failed_recipient) = &failed.failed_recipient {
+                let contact_id =
+                    Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
+                if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+                    // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
+                    chat::add_info_msg(
+                        context,
+                        chat_id,
+                        context
+                            .stock_string_repl_str(
+                                StockMessage::FailedSendingTo,
+                                contact.get_display_name(),
+                            )
+                            .await,
+                    )
+                    .await;
+                    context.emit_event(Event::ChatModified(chat_id));
+                }
+            }
+        }
+    }
+}
+
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
 pub async fn get_real_msg_cnt(context: &Context) -> i32 {
     match context

src/mimefactory.rs

@@ -351,16 +351,47 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    let raw = message::get_summarytext_by_raw(
-                        self.msg.viewtype,
-                        self.msg.text.as_ref(),
-                        &self.msg.param,
-                        32,
-                        self.context,
-                    )
-                    .await;
-                    let raw_subject = raw.lines().next().unwrap_or_default();
-                    format!("Chat: {}", raw_subject)
+                    match chat.param.get(Param::LastSubject) {
+                        Some(last_subject) => {
+                            let subject_start = if last_subject.starts_with("Chat:") {
+                                0
+                            } else {
+                                // "Antw:" is the longest abbreviation in
+                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+                                // so look at the first _5_ characters:
+                                match last_subject.chars().take(5).position(|c| c == ':') {
+                                    Some(prefix_end) => prefix_end + 1,
+                                    None => 0,
+                                }
+                            };
+                            format!(
+                                "Re: {}",
+                                last_subject
+                                    .chars()
+                                    .skip(subject_start)
+                                    .collect::<String>()
+                                    .trim()
+                            )
+                        }
+                        None => {
+                            let self_name = match self.context.get_config(Config::Displayname).await
+                            {
+                                Some(name) => name,
+                                None => self
+                                    .context
+                                    .get_config(Config::Addr)
+                                    .await
+                                    .unwrap_or_default(),
+                            };
+
+                            self.context
+                                .stock_string_repl_str(
+                                    StockMessage::SubjectForNewContact,
+                                    self_name,
+                                )
+                                .await
+                        }
+                    }
                 }
             }
             Loaded::MDN { .. } => self
@@ -409,6 +440,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
+        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -1174,6 +1207,11 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chatlist::Chatlist;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::mimeparser::*;
+    use crate::test_utils::configured_offline_context;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1181,6 +1219,9 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
+        assert!(!display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1192,6 +1233,25 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
+    #[test]
+    fn test_render_email_address_noescape() {
+        let display_name = "a space";
+        let addr = "x@y.org";
+
+        assert!(display_name.is_ascii());
+        assert!(display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+        let s = format!(
+            "{}",
+            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
+        );
+
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        assert_eq!(s, "a space <x@y.org>");
+    }
+
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1234,4 +1294,192 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
+
+    #[async_std::test]
+    async fn test_subject() {
+        // 1.: Receive a mail from an MUA or Delta Chat
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Infos: 42"
+        );
+
+        // 2. Receive a message from Delta Chat when we did not send any messages before
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Charlie <charlie@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        // 3. Send the first message to a new contact
+        let t = configured_offline_context().await;
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
+
+        let t = configured_offline_context().await;
+        t.ctx
+            .set_config(Config::Displayname, Some("Alice"))
+            .await
+            .unwrap();
+        assert_eq!(first_subject_str(t).await, "Message from Alice");
+
+        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.org>\n\
+            To: alice@example.org\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.org>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.org>\n\
+            To: alice@example.org\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.org>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+    }
+
+    async fn first_subject_str(t: TestContext) -> String {
+        let contact_id =
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.org", Origin::ManuallyCreated)
+                .await
+                .unwrap()
+                .0;
+
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+
+        mf.subject_str().await
+    }
+
+    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+        let t = configured_offline_context().await;
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        mf.subject_str().await
+    }
+
+    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+        context
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(context, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        new_msg
+    }
+
+    #[async_std::test]
+    // This test could still be extended
+    async fn test_render_reply() {
+        let t = configured_offline_context().await;
+        let context = &t.ctx;
+
+        let msg = incoming_msg_to_reply_msg(
+            b"From: Charlie <charlie@example.org>\n\
+                To: alice@example.org\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.org>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            context,
+        )
+        .await;
+
+        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+
+        let recipients = mimefactory.recipients();
+        assert_eq!(recipients, vec!["charlie@example.org"]);
+
+        let rendered_msg = mimefactory.render().await.unwrap();
+
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        assert_eq!(
+            mail.headers
+                .iter()
+                .find(|h| h.get_key() == "MIME-Version")
+                .unwrap()
+                .get_value(),
+            "1.0"
+        );
+
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+            .await
+            .unwrap();
+    }
 }

src/mimeparser.rs

@@ -3,6 +3,7 @@ use std::future::Future;
 use std::pin::Pin;
 
 use deltachat_derive::{FromSql, ToSql};
+use lazy_static::lazy_static;
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 
@@ -17,6 +18,7 @@ use crate::e2ee;
 use crate::error::{bail, Result};
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
 use crate::param::*;
@@ -44,7 +46,7 @@ pub struct MimeMessage {
     pub from: Vec<SingleInfo>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
-    pub signatures: HashSet<String>,
+    pub signatures: HashSet<Fingerprint>,
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
@@ -52,7 +54,8 @@ pub struct MimeMessage {
     pub message_kml: Option<location::Kml>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
-    pub(crate) reports: Vec<Report>,
+    pub(crate) mdn_reports: Vec<Report>,
+    pub(crate) failure_report: Option<FailureReport>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -175,14 +178,16 @@ impl MimeMessage {
             signatures,
             gossipped_addr,
             is_forwarded: false,
-            reports: Vec::new(),
+            mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
             user_avatar: None,
             group_avatar: None,
+            failure_report: None,
         };
         parser.parse_mime_recursive(context, &mail).await?;
+        parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
         Ok(parser)
@@ -252,7 +257,8 @@ impl MimeMessage {
                 self.parts[0].msg = "".to_string();
 
                 // swap new with old
-                std::mem::replace(&mut self.parts[0], filepart);
+                self.parts.push(filepart); // push to the end
+                let _ = self.parts.swap_remove(0); // drops first element, replacing it with the last one in O(1)
             }
         }
     }
@@ -352,7 +358,7 @@ impl MimeMessage {
         // just have send a message in the subject with an empty body.
         // Besides, we want to show something in case our incoming-processing
         // failed to properly handle an incoming message.
-        if self.parts.is_empty() && self.reports.is_empty() {
+        if self.parts.is_empty() && self.mdn_reports.is_empty() {
             let mut part = Part::default();
             part.typ = Viewtype::Text;
 
@@ -526,6 +532,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
+                part.error = "Decryption failed".to_string();
 
                 self.parts.push(part);
 
@@ -548,10 +555,10 @@ impl MimeMessage {
             (mime::MULTIPART, "report") => {
                 /* RFC 6522: the first part is for humans, the second for machines */
                 if mail.subparts.len() >= 2 {
-                    if let Some(report_type) = mail.ctype.params.get("report-type") {
-                        if report_type == "disposition-notification" {
+                    match mail.ctype.params.get("report-type").map(|s| s as &str) {
+                        Some("disposition-notification") => {
                             if let Some(report) = self.process_report(context, mail)? {
-                                self.reports.push(report);
+                                self.mdn_reports.push(report);
                             }
 
                             // Add MDN part so we can track it, avoid
@@ -563,9 +570,21 @@ impl MimeMessage {
                             self.parts.push(part);
 
                             any_part_added = true;
-                        } else {
-                            /* eg. `report-type=delivery-status`;
-                            maybe we should show them as a little error icon */
+                        }
+                        // Some providers, e.g. Tiscali, forget to set the report-type. So, if it's None, assume that it might be delivery-status
+                        Some("delivery-status") | None => {
+                            if let Some(report) = self.process_delivery_status(context, mail)? {
+                                self.failure_report = Some(report);
+                            }
+
+                            // Add all parts (we need another part, preferrably text/plain, to show as an error message)
+                            for cur_data in mail.subparts.iter() {
+                                if self.parse_mime_recursive(context, cur_data).await? {
+                                    any_part_added = true;
+                                }
+                            }
+                        }
+                        Some(_) => {
                             if let Some(first) = mail.subparts.iter().next() {
                                 any_part_added = self.parse_mime_recursive(context, first).await?;
                             }
@@ -838,24 +857,118 @@ impl MimeMessage {
         Ok(None)
     }
 
-    /// Handle reports (only MDNs for now)
-    pub async fn handle_reports(&self, context: &Context, from_id: u32, sent_timestamp: i64) {
-        if self.reports.is_empty() {
-            return;
+    fn process_delivery_status(
+        &self,
+        context: &Context,
+        report: &mailparse::ParsedMail<'_>,
+    ) -> Result<Option<FailureReport>> {
+        // parse as mailheaders
+        if let Some(original_msg) = report
+            .subparts
+            .iter()
+            .find(|p| p.ctype.mimetype.contains("rfc822") || p.ctype.mimetype == "message/global")
+        {
+            let report_body = original_msg.get_body_raw()?;
+            let (report_fields, _) = mailparse::parse_headers(&report_body)?;
+
+            if let Some(original_message_id) = report_fields
+                .get_header_value(HeaderDef::MessageId)
+                .and_then(|v| parse_message_id(&v).ok())
+            {
+                let mut to_list = get_all_addresses_from_header(&report.headers, |header_key| {
+                    header_key == "x-failed-recipients"
+                });
+                let to = if to_list.len() == 1 {
+                    Some(to_list.pop().unwrap())
+                } else {
+                    None // We do not know which recipient failed
+                };
+
+                return Ok(Some(FailureReport {
+                    rfc724_mid: original_message_id,
+                    failed_recipient: to.map(|s| s.addr),
+                }));
+            }
+
+            warn!(
+                context,
+                "ignoring unknown ndn-notification, Message-Id: {:?}",
+                report_fields.get_header_value(HeaderDef::MessageId)
+            );
         }
 
-        for report in &self.reports {
+        Ok(None)
+    }
+
+    /// Some providers like GMX and Yahoo do not send standard NDNs (Non Delivery notifications).
+    /// If you improve heuristics here you might also have to change prefetch_should_download() in imap/mod.rs.
+    /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
+    async fn heuristically_parse_ndn(&mut self, context: &Context) -> Option<()> {
+        let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
+            let from = from.to_ascii_lowercase();
+            from.contains("mailer-daemon") || from.contains("mail-daemon")
+        } else {
+            false
+        };
+        if maybe_ndn && self.failure_report.is_none() {
+            lazy_static! {
+                static ref RE: regex::Regex = regex::Regex::new(r"Message-ID:(.*)").unwrap();
+            }
+            for captures in self
+                .parts
+                .iter()
+                .filter_map(|part| part.msg_raw.as_ref())
+                .flat_map(|part| part.lines())
+                .filter_map(|line| RE.captures(line))
+            {
+                if let Ok(original_message_id) = parse_message_id(&captures[1]) {
+                    if let Ok(Some(_)) =
+                        message::rfc724_mid_exists(context, &original_message_id).await
+                    {
+                        self.failure_report = Some(FailureReport {
+                            rfc724_mid: original_message_id,
+                            failed_recipient: None,
+                        })
+                    }
+                }
+            }
+        }
+        None // Always return None, we just return anything so that we can use the '?' operator.
+    }
+
+    /// Handle reports
+    /// (MDNs = Message Disposition Notification, the message was read
+    /// and NDNs = Non delivery notification, the message could not be delivered)
+    pub async fn handle_reports(
+        &self,
+        context: &Context,
+        from_id: u32,
+        sent_timestamp: i64,
+        parts: &[Part],
+    ) {
+        for report in &self.mdn_reports {
             for original_message_id in
                 std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
             {
                 if let Some((chat_id, msg_id)) =
-                    message::mdn_from_ext(context, from_id, original_message_id, sent_timestamp)
-                        .await
+                    message::handle_mdn(context, from_id, original_message_id, sent_timestamp).await
                 {
                     context.emit_event(Event::MsgRead { chat_id, msg_id });
                 }
             }
         }
+
+        if let Some(failure_report) = &self.failure_report {
+            let error = parts.iter().find(|p| p.typ == Viewtype::Text).map(|p| {
+                let msg = &p.msg;
+                match msg.find("\n--- ") {
+                    Some(footer_start) => &msg[..footer_start],
+                    None => msg,
+                }
+                .trim()
+            });
+            message::handle_ndn(context, failure_report, error).await
+        }
     }
 }
 
@@ -912,6 +1025,12 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
+#[derive(Debug)]
+pub(crate) struct FailureReport {
+    pub rfc724_mid: String,
+    pub failed_recipient: Option<String>,
+}
+
 pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
     // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
     let mut msgids = Vec::new();
@@ -955,6 +1074,7 @@ pub struct Part {
     pub bytes: usize,
     pub param: Params,
     org_filename: Option<String>,
+    pub error: String,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1401,7 +1521,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
+        assert_eq!(message.mdn_reports.len(), 1);
     }
 
     /// Test parsing multiple MDNs combined in a single message.
@@ -1481,7 +1601,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 2);
-        assert_eq!(message.reports.len(), 2);
+        assert_eq!(message.mdn_reports.len(), 2);
     }
 
     #[async_std::test]
@@ -1528,10 +1648,13 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
-        assert_eq!(message.reports[0].original_message_id, "foo@example.org");
+        assert_eq!(message.mdn_reports.len(), 1);
         assert_eq!(
-            &message.reports[0].additional_message_ids,
+            message.mdn_reports[0].original_message_id,
+            "foo@example.org"
+        );
+        assert_eq!(
+            &message.mdn_reports[0].additional_message_ids,
             &["foo@example.com", "foo@example.net"]
         );
     }

src/oauth2.rs

@@ -1,7 +1,9 @@
 //! OAuth 2 module
 
+use regex::Regex;
 use std::collections::HashMap;
 
+use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
@@ -15,6 +17,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
+    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -24,8 +27,11 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
+    mx_pattern: None,
 };
 
+const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -33,6 +39,7 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
+    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -53,7 +60,7 @@ pub async fn dc_get_oauth2_url(
     addr: impl AsRef<str>,
     redirect_uri: impl AsRef<str>,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
             .set_raw_config(
@@ -81,7 +88,7 @@ pub async fn dc_get_oauth2_access_token(
     code: impl AsRef<str>,
     regenerate: bool,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -239,7 +246,7 @@ pub async fn dc_get_oauth2_addr(
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    let oauth2 = Oauth2::from_address(addr.as_ref()).await?;
     oauth2.get_userinfo?;
 
     if let Some(access_token) =
@@ -263,23 +270,62 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    fn from_address(addr: impl AsRef<str>) -> Option<Self> {
+    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
         let addr_normalized = normalize_addr(addr.as_ref());
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            match domain {
-                "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
-                | "narod.ru" => Some(OAUTH2_YANDEX),
-                _ => None,
+            if let Some(provider) = Oauth2::lookup_whitelist(domain) {
+                Some(provider)
+            } else {
+                Oauth2::lookup_mx(domain).await
             }
         } else {
             None
         }
     }
 
+    fn lookup_whitelist(domain: impl AsRef<str>) -> Option<Self> {
+        let domain = domain.as_ref();
+        match domain {
+            "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
+            "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
+            | "narod.ru" => Some(OAUTH2_YANDEX),
+            _ => None,
+        }
+    }
+
+    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
+        if let Ok(resolver) = resolver(
+            config::ResolverConfig::default(),
+            config::ResolverOpts::default(),
+        )
+        .await
+        {
+            for provider in OAUTH2_PROVIDERS.iter() {
+                if let Some(pattern) = provider.mx_pattern {
+                    let re = Regex::new(pattern).unwrap();
+
+                    let mut fqdn: String = String::from(domain.as_ref());
+                    if !fqdn.ends_with('.') {
+                        fqdn.push_str(".");
+                    }
+
+                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
+                        for rr in res.iter() {
+                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
+                                return Some(provider.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
         let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
@@ -362,20 +408,34 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_oauth_from_address() {
-        assert_eq!(Oauth2::from_address("hello@gmail.com"), Some(OAUTH2_GMAIL));
+    #[async_std::test]
+    async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com"),
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com"),
+            Oauth2::from_address("hello@googlemail.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.com").await,
+            Some(OAUTH2_YANDEX)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address("hello@yandex.ru"), Some(OAUTH2_YANDEX));
 
-        assert_eq!(Oauth2::from_address("hello@web.de"), None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
+    }
+
+    #[async_std::test]
+    async fn test_oauth_from_mx() {
+        assert_eq!(
+            Oauth2::from_address("hello@google.com").await,
+            Some(OAUTH2_GMAIL)
+        );
     }
 
     #[async_std::test]

src/param.rs

@@ -65,9 +65,6 @@ pub enum Param {
     /// For Messages
     Arg4 = b'H',
 
-    /// For Messages
-    Error = b'L',
-
     /// For Messages
     AttachGroupImage = b'A',
 
@@ -103,6 +100,9 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
+    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    LastSubject = b't',
+
     /// For Chats
     Devicetalk = b'D',
 

src/peerstate.rs

@@ -7,7 +7,7 @@ use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::context::Context;
-use crate::key::{DcKey, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
 
 #[derive(Debug)]
@@ -32,12 +32,12 @@ pub struct Peerstate<'a> {
     pub last_seen_autocrypt: i64,
     pub prefer_encrypt: EncryptPreference,
     pub public_key: Option<SignedPublicKey>,
-    pub public_key_fingerprint: Option<String>,
+    pub public_key_fingerprint: Option<Fingerprint>,
     pub gossip_key: Option<SignedPublicKey>,
     pub gossip_timestamp: i64,
-    pub gossip_key_fingerprint: Option<String>,
+    pub gossip_key_fingerprint: Option<Fingerprint>,
     pub verified_key: Option<SignedPublicKey>,
-    pub verified_key_fingerprint: Option<String>,
+    pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
     pub degrade_event: Option<DegradeEvent>,
 }
@@ -151,7 +151,7 @@ impl<'a> Peerstate<'a> {
     pub async fn from_fingerprint(
         context: &'a Context,
         _sql: &Sql,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
     ) -> Option<Peerstate<'a>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
@@ -160,13 +160,8 @@ impl<'a> Peerstate<'a> {
                      WHERE public_key_fingerprint=? COLLATE NOCASE \
                      OR gossip_key_fingerprint=? COLLATE NOCASE  \
                      ORDER BY public_key_fingerprint=? DESC;";
-
-        Self::from_stmt(
-            context,
-            query,
-            paramsv![fingerprint, fingerprint, fingerprint],
-        )
-        .await
+        let fp = fingerprint.hex();
+        Self::from_stmt(context, query, paramsv![fp, fp, fp]).await
     }
 
     async fn from_stmt(
@@ -189,33 +184,18 @@ impl<'a> Peerstate<'a> {
                 res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
                 res.gossip_timestamp = row.get(5)?;
 
-                res.public_key_fingerprint = row.get(7)?;
-                if res
-                    .public_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.public_key_fingerprint = None;
-                }
-                res.gossip_key_fingerprint = row.get(8)?;
-                if res
-                    .gossip_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.gossip_key_fingerprint = None;
-                }
-                res.verified_key_fingerprint = row.get(10)?;
-                if res
-                    .verified_key_fingerprint
-                    .as_ref()
-                    .map(|s| s.is_empty())
-                    .unwrap_or_default()
-                {
-                    res.verified_key_fingerprint = None;
-                }
+                res.public_key_fingerprint = row
+                    .get::<_, Option<String>>(7)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.gossip_key_fingerprint = row
+                    .get::<_, Option<String>>(8)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
+                res.verified_key_fingerprint = row
+                    .get::<_, Option<String>>(10)?
+                    .map(|s| s.parse::<Fingerprint>())
+                    .transpose()?;
                 res.public_key = row
                     .get(4)
                     .ok()
@@ -238,7 +218,7 @@ impl<'a> Peerstate<'a> {
     pub fn recalc_fingerprint(&mut self) {
         if let Some(ref public_key) = self.public_key {
             let old_public_fingerprint = self.public_key_fingerprint.take();
-            self.public_key_fingerprint = Some(public_key.fingerprint().hex());
+            self.public_key_fingerprint = Some(public_key.fingerprint());
 
             if old_public_fingerprint.is_none()
                 || self.public_key_fingerprint.is_none()
@@ -253,7 +233,7 @@ impl<'a> Peerstate<'a> {
 
         if let Some(ref gossip_key) = self.gossip_key {
             let old_gossip_fingerprint = self.gossip_key_fingerprint.take();
-            self.gossip_key_fingerprint = Some(gossip_key.fingerprint().hex());
+            self.gossip_key_fingerprint = Some(gossip_key.fingerprint());
 
             if old_gossip_fingerprint.is_none()
                 || self.gossip_key_fingerprint.is_none()
@@ -387,7 +367,7 @@ impl<'a> Peerstate<'a> {
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
-        fingerprint: &str,
+        fingerprint: &Fingerprint,
         verified: PeerstateVerifiedStatus,
     ) -> bool {
         if verified == PeerstateVerifiedStatus::BidirectVerified {
@@ -445,10 +425,10 @@ impl<'a> Peerstate<'a> {
                     self.public_key.as_ref().map(|k| k.to_bytes()),
                     self.gossip_timestamp,
                     self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    self.public_key_fingerprint,
-                    self.gossip_key_fingerprint,
+                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    self.verified_key_fingerprint,
+                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
                     self.addr,
                 ],
             ).await?;
@@ -469,15 +449,18 @@ impl<'a> Peerstate<'a> {
         Ok(())
     }
 
-    pub fn has_verified_key(&self, fingerprints: &HashSet<String>) -> bool {
-        if self.verified_key.is_some() && self.verified_key_fingerprint.is_some() {
-            let vkc = self.verified_key_fingerprint.as_ref().unwrap();
-            if fingerprints.contains(vkc) {
-                return true;
-            }
+    pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
+        if let Some(vkc) = &self.verified_key_fingerprint {
+            fingerprints.contains(vkc) && self.verified_key.is_some()
+        } else {
+            false
         }
+    }
+}
 
-        false
+impl From<crate::key::FingerprintError> for rusqlite::Error {
+    fn from(_source: crate::key::FingerprintError) -> Self {
+        Self::InvalidColumnType(0, "Invalid fingerprint".into(), rusqlite::types::Type::Text)
     }
 }
 
@@ -502,12 +485,12 @@ mod tests {
             last_seen_autocrypt: 11,
             prefer_encrypt: EncryptPreference::Mutual,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
             gossip_key: Some(pub_key.clone()),
             gossip_timestamp: 12,
-            gossip_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            gossip_key_fingerprint: Some(pub_key.fingerprint()),
             verified_key: Some(pub_key.clone()),
-            verified_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            verified_key_fingerprint: Some(pub_key.fingerprint()),
             to_save: Some(ToSave::All),
             degrade_event: None,
         };
@@ -525,7 +508,7 @@ mod tests {
         peerstate.to_save = None;
         assert_eq!(peerstate, peerstate_new);
         let peerstate_new2 =
-            Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint().hex())
+            Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint())
                 .await
                 .expect("failed to load peerstate from db");
         assert_eq!(peerstate, peerstate_new2);
@@ -544,7 +527,7 @@ mod tests {
             last_seen_autocrypt: 11,
             prefer_encrypt: EncryptPreference::Mutual,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
             gossip_key: None,
             gossip_timestamp: 12,
             gossip_key_fingerprint: None,
@@ -578,7 +561,7 @@ mod tests {
             last_seen_autocrypt: 11,
             prefer_encrypt: EncryptPreference::Mutual,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint().hex()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
             gossip_key: None,
             gossip_timestamp: 12,
             gossip_key_fingerprint: None,

src/pgp.rs

@@ -18,7 +18,7 @@ use rand::{thread_rng, CryptoRng, Rng};
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::key::DcKey;
+use crate::key::{DcKey, Fingerprint};
 use crate::keyring::Keyring;
 
 pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
@@ -272,12 +272,20 @@ pub async fn pk_encrypt(
     .await
 }
 
+/// Decrypts the message with keys from the private key keyring.
+///
+/// Receiver private keys are provided in
+/// `private_keys_for_decryption`.
+///
+/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// of all keys from the `public_keys_for_validation` keyring that
+/// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
 pub async fn pk_decrypt(
     ctext: Vec<u8>,
     private_keys_for_decryption: Keyring<SignedSecretKey>,
     public_keys_for_validation: Keyring<SignedPublicKey>,
-    ret_signature_fingerprints: Option<&mut HashSet<String>>,
+    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
 ) -> Result<Vec<u8>> {
     let msgs = async_std::task::spawn_blocking(move || {
         let cursor = Cursor::new(ctext);
@@ -290,36 +298,41 @@ pub async fn pk_decrypt(
     })
     .await?;
 
-    ensure!(!msgs.is_empty(), "No valid messages found");
+    if let Some(msg) = msgs.into_iter().next() {
+        // get_content() will decompress the message if needed,
+        // but this avoids decompressing it again to check signatures
+        let msg = msg.decompress()?;
 
-    let content = match msgs[0].get_content()? {
-        Some(content) => content,
-        None => bail!("Decrypted message is empty"),
-    };
+        let content = match msg.get_content()? {
+            Some(content) => content,
+            None => bail!("The decrypted message is empty"),
+        };
 
-    if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-        if !public_keys_for_validation.is_empty() {
-            let fingerprints = async_std::task::spawn_blocking(move || {
-                let dec_msg = &msgs[0];
+        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
+            if !public_keys_for_validation.is_empty() {
+                let fingerprints = async_std::task::spawn_blocking(move || {
+                    let pkeys = public_keys_for_validation.keys();
 
-                let pkeys = public_keys_for_validation.keys();
-
-                let mut fingerprints = Vec::new();
-                for pkey in pkeys {
-                    if dec_msg.verify(&pkey.primary_key).is_ok() {
-                        let fp = DcKey::fingerprint(pkey).hex();
-                        fingerprints.push(fp);
+                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                        for pkey in pkeys {
+                            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                                let fp = DcKey::fingerprint(pkey);
+                                fingerprints.push(fp);
+                            }
+                        }
                     }
-                }
-                fingerprints
-            })
-            .await;
+                    fingerprints
+                })
+                .await;
 
-            ret_signature_fingerprints.extend(fingerprints);
+                ret_signature_fingerprints.extend(fingerprints);
+            }
         }
+        Ok(content)
+    } else {
+        bail!("No valid messages found");
     }
-
-    Ok(content)
 }
 
 /// Symmetric encryption.
@@ -474,7 +487,7 @@ mod tests {
         decrypt_keyring.add(KEYS.alice_secret.clone());
         let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
@@ -492,7 +505,7 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
@@ -511,7 +524,7 @@ mod tests {
         let mut keyring = Keyring::new();
         keyring.add(KEYS.alice_secret.clone());
         let empty_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             keyring,
@@ -531,7 +544,7 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.bob_public.clone());
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
@@ -549,7 +562,7 @@ mod tests {
         let mut decrypt_keyring = Keyring::new();
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let sig_check_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<String> = Default::default();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
         let plain = pk_decrypt(
             CTEXT_UNSIGNED.as_bytes().to_vec(),
             decrypt_keyring,

src/provider/data.rs

@@ -19,6 +19,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // aol.md: aol.com
@@ -30,6 +31,7 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // autistici.org.md: autistici.org
@@ -43,6 +45,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -56,6 +59,21 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
+    };
+
+    // chello.at.md: chello.at
+    static ref P_CHELLO_AT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/chello-at",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "mail.mymagenta.at", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.mymagenta.at", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
     };
 
     // comcast.md: xfinity.com, comcast.net
@@ -65,7 +83,16 @@ lazy_static::lazy_static! {
     // - skipping provider with status OK and no special things to do
 
     // disroot.md: disroot.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISROOT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/disroot",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // example.com.md: example.com, example.org
     static ref P_EXAMPLE_COM: Provider = Provider {
@@ -78,6 +105,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // fastmail.md: fastmail.com
@@ -89,6 +117,19 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
+    };
+
+    // five.chat.md: five.chat
+    static ref P_FIVE_CHAT: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/five-chat",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
     };
 
     // freenet.de.md: freenet.de
@@ -102,6 +143,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -115,6 +157,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -129,6 +172,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // i.ua.md: i.ua
@@ -145,6 +189,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // kolst.com.md: kolst.com
@@ -157,7 +202,16 @@ lazy_static::lazy_static! {
     // - skipping provider with status OK and no special things to do
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAILBOX_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mailbox-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // nauta.cu.md: nauta.cu
     static ref P_NAUTA_CU: Provider = Provider {
@@ -178,6 +232,7 @@ lazy_static::lazy_static! {
             ConfigDefault { key: Config::E2eeEnabled, value: "0" },
             ConfigDefault { key: Config::MediaQuality, value: "1" },
         ]),
+        strict_tls: false,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -191,9 +246,10 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
-    // posteo.md: posteo.de
+    // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
     static ref P_POSTEO: Provider = Provider {
         status: Status::OK,
         before_login_hint: "",
@@ -204,6 +260,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -215,16 +272,35 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // riseup.net.md: riseup.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_RISEUP_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/riseup-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // rogers.com.md: rogers.com
     // - skipping provider with status OK and no special things to do
 
     // systemli.org.md: systemli.org
-    // - skipping provider with status OK and no special things to do
+    static ref P_SYSTEMLI_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/systemli-org",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: true,
+    };
 
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
@@ -235,6 +311,7 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // testrun.md: testrun.org
@@ -249,6 +326,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // tiscali.it.md: tiscali.it
@@ -262,6 +340,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // ukr.net.md: ukr.net
@@ -282,12 +361,13 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
     static ref P_YAHOO: Provider = Provider {
         status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with your Yahoo email address you have to allow \"less secure apps\" in the Yahoo webinterface.",
+        before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yahoo",
         server: vec![
@@ -295,6 +375,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     // yandex.ru.md: yandex.ru, yandex.com
@@ -306,6 +387,7 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
+        strict_tls: true,
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -319,6 +401,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
+        strict_tls: false,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
@@ -326,9 +409,12 @@ lazy_static::lazy_static! {
         ("aol.com", &*P_AOL),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("chello.at", &*P_CHELLO_AT),
+        ("disroot.org", &*P_DISROOT),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -344,6 +430,8 @@ lazy_static::lazy_static! {
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
@@ -351,8 +439,58 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
+        ("posteo.af", &*P_POSTEO),
+        ("posteo.at", &*P_POSTEO),
+        ("posteo.be", &*P_POSTEO),
+        ("posteo.ch", &*P_POSTEO),
+        ("posteo.cl", &*P_POSTEO),
+        ("posteo.co", &*P_POSTEO),
+        ("posteo.co.uk", &*P_POSTEO),
+        ("posteo.com.br", &*P_POSTEO),
+        ("posteo.cr", &*P_POSTEO),
+        ("posteo.cz", &*P_POSTEO),
+        ("posteo.dk", &*P_POSTEO),
+        ("posteo.ee", &*P_POSTEO),
+        ("posteo.es", &*P_POSTEO),
+        ("posteo.eu", &*P_POSTEO),
+        ("posteo.fi", &*P_POSTEO),
+        ("posteo.gl", &*P_POSTEO),
+        ("posteo.gr", &*P_POSTEO),
+        ("posteo.hn", &*P_POSTEO),
+        ("posteo.hr", &*P_POSTEO),
+        ("posteo.hu", &*P_POSTEO),
+        ("posteo.ie", &*P_POSTEO),
+        ("posteo.in", &*P_POSTEO),
+        ("posteo.is", &*P_POSTEO),
+        ("posteo.jp", &*P_POSTEO),
+        ("posteo.la", &*P_POSTEO),
+        ("posteo.li", &*P_POSTEO),
+        ("posteo.lt", &*P_POSTEO),
+        ("posteo.lu", &*P_POSTEO),
+        ("posteo.me", &*P_POSTEO),
+        ("posteo.mx", &*P_POSTEO),
+        ("posteo.my", &*P_POSTEO),
+        ("posteo.net", &*P_POSTEO),
+        ("posteo.nl", &*P_POSTEO),
+        ("posteo.no", &*P_POSTEO),
+        ("posteo.nz", &*P_POSTEO),
+        ("posteo.org", &*P_POSTEO),
+        ("posteo.pe", &*P_POSTEO),
+        ("posteo.pl", &*P_POSTEO),
+        ("posteo.pm", &*P_POSTEO),
+        ("posteo.pt", &*P_POSTEO),
+        ("posteo.ro", &*P_POSTEO),
+        ("posteo.ru", &*P_POSTEO),
+        ("posteo.se", &*P_POSTEO),
+        ("posteo.sg", &*P_POSTEO),
+        ("posteo.si", &*P_POSTEO),
+        ("posteo.tn", &*P_POSTEO),
+        ("posteo.uk", &*P_POSTEO),
+        ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),

src/provider/mod.rs

@@ -72,6 +72,7 @@ pub struct Provider {
     pub overview_page: &'static str,
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
+    pub strict_tls: bool,
 }
 
 impl Provider {

src/provider/update.py

@@ -100,6 +100,9 @@ def process_data(data, file):
 
     config_defaults = process_config_defaults(data)
 
+    strict_tls = data.get("strict_tls", False)
+    strict_tls = "true" if strict_tls else "false"
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -111,6 +114,7 @@ def process_data(data, file):
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
         provider += "        config_defaults: " + config_defaults + ",\n"
+        provider += "        strict_tls: " + strict_tls + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -121,7 +125,7 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
+    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None" and strict_tls == "false":
         out_all += "    // - skipping provider with status OK and no special things to do\n\n"
     else:
         out_all += provider

src/qr.rs

@@ -10,8 +10,7 @@ use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::key::dc_format_fingerprint;
-use crate::key::dc_normalize_fingerprint;
+use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
@@ -80,6 +79,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         Some(pair) => pair,
         None => (payload, ""),
     };
+    let fingerprint: Fingerprint = match fingerprint.parse() {
+        Ok(fp) => fp,
+        Err(err) => {
+            return Error::new(err)
+                .context("Failed to parse fingerprint in QR code")
+                .into()
+        }
+    };
 
     // replace & with \n to match expected param format
     let fragment = fragment.replace('&', "\n");
@@ -128,13 +135,6 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         None
     };
 
-    let fingerprint = dc_normalize_fingerprint(fingerprint);
-
-    // ensure valid fingerprint
-    if fingerprint.len() != 40 {
-        return format_err!("Bad fingerprint length in QR code").into();
-    }
-
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
@@ -161,7 +161,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             chat::add_info_msg(context, id, format!("{} verified.", peerstate.addr)).await;
         } else {
             lot.state = LotState::QrFprWithoutAddr;
-            lot.text1 = Some(dc_format_fingerprint(&fingerprint));
+            lot.text1 = Some(fingerprint.to_string());
         }
     } else if let Some(addr) = addr {
         if grpid.is_some() && grpname.is_some() {

src/scheduler.rs

@@ -5,7 +5,7 @@ use async_std::task;
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::smtp::Smtp;
+use crate::{config::Config, message::MsgId, smtp::Smtp};
 
 pub(crate) struct StopToken;
 
@@ -32,36 +32,20 @@ impl Context {
         self.scheduler.read().await.maybe_network().await;
     }
 
-    pub(crate) async fn interrupt_inbox(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_inbox(probe_network)
-            .await;
+    pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_inbox(info).await;
     }
 
-    pub(crate) async fn interrupt_sentbox(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_sentbox(probe_network)
-            .await;
+    pub(crate) async fn interrupt_sentbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_sentbox(info).await;
     }
 
-    pub(crate) async fn interrupt_mvbox(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_mvbox(probe_network)
-            .await;
+    pub(crate) async fn interrupt_mvbox(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_mvbox(info).await;
     }
 
-    pub(crate) async fn interrupt_smtp(&self, probe_network: bool) {
-        self.scheduler
-            .read()
-            .await
-            .interrupt_smtp(probe_network)
-            .await;
+    pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
+        self.scheduler.read().await.interrupt_smtp(info).await;
     }
 }
 
@@ -79,20 +63,16 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         // track number of continously executed jobs
         let mut jobs_loaded = 0;
-        let mut probe_network = false;
+        let mut info = InterruptInfo::default();
         loop {
-            match job::load_next(&ctx, Thread::Imap, probe_network).await {
+            match job::load_next(&ctx, Thread::Imap, &info).await {
                 Some(job) if jobs_loaded <= 20 => {
                     jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
-                    probe_network = false;
+                    info = Default::default();
                 }
                 Some(job) => {
                     // Let the fetch run, but return back to the job afterwards.
@@ -102,7 +82,7 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 }
                 None => {
                     jobs_loaded = 0;
-                    probe_network = fetch_idle(&ctx, &mut connection).await;
+                    info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
                 }
             }
         }
@@ -119,15 +99,18 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 }
 
 async fn fetch(ctx: &Context, connection: &mut Imap) {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "{}", err);
+                return;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
         }
         None => {
             warn!(ctx, "Can not fetch inbox folder, not set");
@@ -136,16 +119,20 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     }
 }
 
-async fn fetch_idle(ctx: &Context, connection: &mut Imap) -> bool {
-    match get_watch_folder(&ctx, "configured_inbox_folder").await {
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
+    match ctx.get_config(folder).await {
         Some(watch_folder) => {
+            // connect and fake idle if unable to connect
+            if let Err(err) = connection.connect_configured(&ctx).await {
+                error!(ctx, "imap connection failed: {}", err);
+                return connection.fake_idle(&ctx, None).await;
+            }
+
             // fetch
-            connection
-                .fetch(&ctx, &watch_folder)
-                .await
-                .unwrap_or_else(|err| {
-                    error!(ctx, "{}", err);
-                });
+            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+                connection.trigger_reconnect();
+                warn!(ctx, "{}", err);
+            }
 
             // idle
             if connection.can_idle() {
@@ -153,15 +140,16 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap) -> bool {
                     .idle(&ctx, Some(watch_folder))
                     .await
                     .unwrap_or_else(|err| {
-                        error!(ctx, "{}", err);
-                        false
+                        connection.trigger_reconnect();
+                        warn!(ctx, "{}", err);
+                        InterruptInfo::new(false, None)
                     })
             } else {
                 connection.fake_idle(&ctx, Some(watch_folder)).await
             }
         }
         None => {
-            warn!(ctx, "Can not watch inbox folder, not set");
+            warn!(ctx, "Can not watch {} folder, not set", folder);
             connection.fake_idle(&ctx, None).await
         }
     }
@@ -171,7 +159,7 @@ async fn simple_imap_loop(
     ctx: Context,
     started: Sender<()>,
     inbox_handlers: ImapConnectionHandlers,
-    folder: impl AsRef<str>,
+    folder: Config,
 ) {
     use futures::future::FutureExt;
 
@@ -187,44 +175,9 @@ async fn simple_imap_loop(
     let fut = async move {
         started.send(()).await;
         let ctx = ctx1;
-        if let Err(err) = connection.connect_configured(&ctx).await {
-            error!(ctx, "{}", err);
-            return;
-        }
 
         loop {
-            match get_watch_folder(&ctx, folder.as_ref()).await {
-                Some(watch_folder) => {
-                    // fetch
-                    connection
-                        .fetch(&ctx, &watch_folder)
-                        .await
-                        .unwrap_or_else(|err| {
-                            error!(ctx, "{}", err);
-                        });
-
-                    // idle
-                    if connection.can_idle() {
-                        connection
-                            .idle(&ctx, Some(watch_folder))
-                            .await
-                            .unwrap_or_else(|err| {
-                                error!(ctx, "{}", err);
-                                false
-                            });
-                    } else {
-                        connection.fake_idle(&ctx, Some(watch_folder)).await;
-                    }
-                }
-                None => {
-                    warn!(
-                        &ctx,
-                        "No watch folder found for {}, skipping",
-                        folder.as_ref()
-                    );
-                    connection.fake_idle(&ctx, None).await;
-                }
-            }
+            fetch_idle(&ctx, &mut connection, folder).await;
         }
     };
 
@@ -254,18 +207,18 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         started.send(()).await;
         let ctx = ctx1;
 
-        let mut probe_network = false;
+        let mut interrupt_info = Default::default();
         loop {
-            match job::load_next(&ctx, Thread::Smtp, probe_network).await {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
                 Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
-                    probe_network = false;
+                    interrupt_info = Default::default();
                 }
                 None => {
                     // Fake Idle
                     info!(ctx, "smtp fake idle - started");
-                    probe_network = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
                     info!(ctx, "smtp fake idle - interrupted")
                 }
             }
@@ -317,7 +270,7 @@ impl Scheduler {
                     ctx1,
                     mvbox_start_send,
                     mvbox_handlers,
-                    "configured_mvbox_folder",
+                    Config::ConfiguredMvboxFolder,
                 )
                 .await
             }));
@@ -331,7 +284,7 @@ impl Scheduler {
                     ctx1,
                     sentbox_start_send,
                     sentbox_handlers,
-                    "configured_sentbox_folder",
+                    Config::ConfiguredSentboxFolder,
                 )
                 .await
             }));
@@ -364,34 +317,34 @@ impl Scheduler {
             return;
         }
 
-        self.interrupt_inbox(true)
-            .join(self.interrupt_mvbox(true))
-            .join(self.interrupt_sentbox(true))
-            .join(self.interrupt_smtp(true))
+        self.interrupt_inbox(InterruptInfo::new(true, None))
+            .join(self.interrupt_mvbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(true, None)))
+            .join(self.interrupt_smtp(InterruptInfo::new(true, None)))
             .await;
     }
 
-    async fn interrupt_inbox(&self, probe_network: bool) {
+    async fn interrupt_inbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref inbox, .. } = self {
-            inbox.interrupt(probe_network).await;
+            inbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_mvbox(&self, probe_network: bool) {
+    async fn interrupt_mvbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref mvbox, .. } = self {
-            mvbox.interrupt(probe_network).await;
+            mvbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_sentbox(&self, probe_network: bool) {
+    async fn interrupt_sentbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref sentbox, .. } = self {
-            sentbox.interrupt(probe_network).await;
+            sentbox.interrupt(info).await;
         }
     }
 
-    async fn interrupt_smtp(&self, probe_network: bool) {
+    async fn interrupt_smtp(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref smtp, .. } = self {
-            smtp.interrupt(probe_network).await;
+            smtp.interrupt(info).await;
         }
     }
 
@@ -460,7 +413,7 @@ struct ConnectionState {
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
-    idle_interrupt_sender: Sender<bool>,
+    idle_interrupt_sender: Sender<InterruptInfo>,
 }
 
 impl ConnectionState {
@@ -472,9 +425,9 @@ impl ConnectionState {
         self.shutdown_receiver.recv().await.ok();
     }
 
-    async fn interrupt(&self, probe_network: bool) {
+    async fn interrupt(&self, info: InterruptInfo) {
         // Use try_send to avoid blocking on interrupts.
-        self.idle_interrupt_sender.try_send(probe_network).ok();
+        self.idle_interrupt_sender.try_send(info).ok();
     }
 }
 
@@ -508,8 +461,8 @@ impl SmtpConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self, probe_network: bool) {
-        self.state.interrupt(probe_network).await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -518,12 +471,11 @@ impl SmtpConnectionState {
     }
 }
 
-#[derive(Debug)]
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
     shutdown_sender: Sender<()>,
-    idle_interrupt_receiver: Receiver<bool>,
+    idle_interrupt_receiver: Receiver<InterruptInfo>,
 }
 
 #[derive(Debug)]
@@ -556,8 +508,8 @@ impl ImapConnectionState {
     }
 
     /// Interrupt any form of idle.
-    async fn interrupt(&self, probe_network: bool) {
-        self.state.interrupt(probe_network).await;
+    async fn interrupt(&self, info: InterruptInfo) {
+        self.state.interrupt(info).await;
     }
 
     /// Shutdown this connection completely.
@@ -573,20 +525,17 @@ struct ImapConnectionHandlers {
     shutdown_sender: Sender<()>,
 }
 
-async fn get_watch_folder(context: &Context, config_name: impl AsRef<str>) -> Option<String> {
-    match context
-        .sql
-        .get_raw_config(context, config_name.as_ref())
-        .await
-    {
-        Some(name) => Some(name),
-        None => {
-            if config_name.as_ref() == "configured_inbox_folder" {
-                // initialized with old version, so has not set configured_inbox_folder
-                Some("INBOX".to_string())
-            } else {
-                None
-            }
+#[derive(Default, Debug)]
+pub struct InterruptInfo {
+    pub probe_network: bool,
+    pub msg_id: Option<MsgId>,
+}
+
+impl InterruptInfo {
+    pub fn new(probe_network: bool, msg_id: Option<MsgId>) -> Self {
+        Self {
+            probe_network,
+            msg_id,
         }
     }
 }

src/securejoin.rs

@@ -14,7 +14,7 @@ use crate::e2ee::*;
 use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, DcKey, SignedPublicKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -73,8 +73,6 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     ====   Step 1 in "Setup verified contact" protocol   ====
     =======================================================*/
 
-    let fingerprint: String;
-
     ensure_secret_key_exists(context).await.ok();
 
     // invitenumber will be used to allow starting the handshake,
@@ -95,7 +93,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         .await
         .unwrap_or_default();
 
-    fingerprint = match get_self_fingerprint(context).await {
+    let fingerprint: Fingerprint = match get_self_fingerprint(context).await {
         Some(fp) => fp,
         None => {
             return None;
@@ -116,7 +114,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
 
             Some(format!(
                 "OPENPGP4FPR:{}#a={}&g={}&x={}&i={}&s={}",
-                fingerprint,
+                fingerprint.hex(),
                 self_addr_urlencoded,
                 &group_name_urlencoded,
                 &chat.grpid,
@@ -131,7 +129,11 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         // parameters used: a=n=i=s=
         Some(format!(
             "OPENPGP4FPR:{}#a={}&n={}&i={}&s={}",
-            fingerprint, self_addr_urlencoded, self_name_urlencoded, &invitenumber, &auth,
+            fingerprint.hex(),
+            self_addr_urlencoded,
+            self_name_urlencoded,
+            &invitenumber,
+            &auth,
         ))
     };
 
@@ -140,9 +142,9 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
     qr
 }
 
-async fn get_self_fingerprint(context: &Context) -> Option<String> {
+async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
     match SignedPublicKey::load_self(context).await {
-        Ok(key) => Some(key.fingerprint().hex()),
+        Ok(key) => Some(key.fingerprint()),
         Err(_) => {
             warn!(context, "get_self_fingerprint(): failed to load key");
             None
@@ -249,7 +251,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
             chat_id_2_contact_id(context, contact_chat_id).await,
             400
         );
-        let own_fingerprint = get_self_fingerprint(context).await.unwrap_or_default();
+        let own_fingerprint = get_self_fingerprint(context).await;
 
         // Bob -> Alice
         if let Err(err) = send_handshake_msg(
@@ -261,7 +263,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
                 "vc-request-with-auth"
             },
             get_qr_attr!(context, auth).to_string(),
-            Some(own_fingerprint),
+            own_fingerprint,
             if join_vg {
                 get_qr_attr!(context, text2).to_string()
             } else {
@@ -311,7 +313,7 @@ async fn send_handshake_msg(
     contact_chat_id: ChatId,
     step: &str,
     param2: impl AsRef<str>,
-    fingerprint: Option<String>,
+    fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
 ) -> Result<(), HandshakeError> {
     let mut msg = Message::default();
@@ -328,7 +330,7 @@ async fn send_handshake_msg(
         msg.param.set(Param::Arg2, param2);
     }
     if let Some(fp) = fingerprint {
-        msg.param.set(Param::Arg3, fp);
+        msg.param.set(Param::Arg3, fp.hex());
     }
     if !grpid.as_ref().is_empty() {
         msg.param.set(Param::Arg4, grpid.as_ref());
@@ -360,7 +362,7 @@ async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32
 
 async fn fingerprint_equals_sender(
     context: &Context,
-    fingerprint: impl AsRef<str>,
+    fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
 ) -> bool {
     let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
@@ -368,9 +370,8 @@ async fn fingerprint_equals_sender(
     if contacts.len() == 1 {
         if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
             if let Some(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
-                let fingerprint_normalized = dc_normalize_fingerprint(fingerprint.as_ref());
                 if peerstate.public_key_fingerprint.is_some()
-                    && &fingerprint_normalized == peerstate.public_key_fingerprint.as_ref().unwrap()
+                    && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
                     return true;
                 }
@@ -397,6 +398,8 @@ pub(crate) enum HandshakeError {
     NoSelfAddr,
     #[error("Failed to send message")]
     MsgSendFailed(#[source] Error),
+    #[error("Failed to parse fingerprint")]
+    BadFingerprint(#[from] crate::key::FingerprintError),
 }
 
 /// What to do with a Secure-Join handshake message after it was handled.
@@ -516,10 +519,11 @@ pub(crate) async fn handle_securejoin_handshake(
                 // no error, just aborted somehow or a mail from another handshake
                 return Ok(HandshakeMessage::Ignore);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
             let auth = get_qr_attr!(context, auth).to_string();
 
-            if !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice) {
+            if !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -576,8 +580,9 @@ pub(crate) async fn handle_securejoin_handshake(
             ==========================================================*/
 
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -588,7 +593,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !encrypted_and_signed(context, mime_message, &fingerprint) {
+            if !encrypted_and_signed(context, mime_message, Some(&fingerprint)) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -625,7 +630,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -673,7 +678,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     contact_chat_id,
                     "vc-contact-confirm",
                     "",
-                    Some(fingerprint.clone()),
+                    Some(fingerprint),
                     "",
                 )
                 .await?;
@@ -709,7 +714,8 @@ pub(crate) async fn handle_securejoin_handshake(
                 );
                 return Ok(abort_retval);
             }
-            let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
+            let scanned_fingerprint_of_alice: Fingerprint =
+                get_qr_attr!(context, fingerprint).clone();
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
@@ -731,7 +737,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 true
             };
             if vg_expect_encrypted
-                && !encrypted_and_signed(context, mime_message, &scanned_fingerprint_of_alice)
+                && !encrypted_and_signed(context, mime_message, Some(&scanned_fingerprint_of_alice))
             {
                 could_not_establish_secure_connection(
                     context,
@@ -888,7 +894,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
             if !encrypted_and_signed(
                 context,
                 mime_message,
-                get_self_fingerprint(context).await.unwrap_or_default(),
+                get_self_fingerprint(context).await.as_ref(),
             ) {
                 could_not_establish_secure_connection(
                     context,
@@ -898,8 +904,9 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 .await;
                 return Ok(HandshakeMessage::Ignore);
             }
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
                 None => {
                     could_not_establish_secure_connection(
                         context,
@@ -910,7 +917,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+            if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
@@ -967,16 +974,13 @@ async fn could_not_establish_secure_connection(
     error!(context, "{} ({})", &msg, details);
 }
 
-async fn mark_peer_as_verified(
-    context: &Context,
-    fingerprint: impl AsRef<str>,
-) -> Result<(), Error> {
+async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
     if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint.as_ref()).await
+        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await
     {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
-            fingerprint.as_ref(),
+            fingerprint,
             PeerstateVerifiedStatus::BidirectVerified,
         ) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
@@ -990,7 +994,7 @@ async fn mark_peer_as_verified(
     }
     bail!(
         "could not mark peer as verified for fingerprint {}",
-        fingerprint.as_ref()
+        fingerprint.hex()
     );
 }
 
@@ -1001,7 +1005,7 @@ async fn mark_peer_as_verified(
 fn encrypted_and_signed(
     context: &Context,
     mimeparser: &MimeMessage,
-    expected_fingerprint: impl AsRef<str>,
+    expected_fingerprint: Option<&Fingerprint>,
 ) -> bool {
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
@@ -1009,17 +1013,17 @@ fn encrypted_and_signed(
     } else if mimeparser.signatures.is_empty() {
         warn!(context, "Message not signed.",);
         false
-    } else if expected_fingerprint.as_ref().is_empty() {
-        warn!(context, "Fingerprint for comparison missing.",);
+    } else if expected_fingerprint.is_none() {
+        warn!(context, "Fingerprint for comparison missing.");
         false
     } else if !mimeparser
         .signatures
-        .contains(expected_fingerprint.as_ref())
+        .contains(expected_fingerprint.unwrap())
     {
         warn!(
             context,
             "Message does not match expected fingerprint {}.",
-            expected_fingerprint.as_ref(),
+            expected_fingerprint.unwrap(),
         );
         false
     } else {

src/smtp/mod.rs

@@ -10,8 +10,9 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
-use crate::login_param::{dc_build_tls, LoginParam};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::oauth2::*;
+use crate::provider::get_provider_info;
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -44,9 +45,8 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Default, DebugStub)]
-pub struct Smtp {
-    #[debug_stub(some = "SmtpTransport")]
+#[derive(Default)]
+pub(crate) struct Smtp {
     transport: Option<smtp::SmtpTransport>,
 
     /// Email address we are sending from.
@@ -113,7 +113,14 @@ impl Smtp {
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls_config = dc_build_tls(lp.smtp_certificate_checks);
+        let provider = get_provider_info(&lp.addr);
+        let strict_tls = match lp.smtp_certificate_checks {
+            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Strict => true,
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => false,
+        };
+        let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
@@ -172,7 +179,7 @@ impl Smtp {
                 .stock_string_repl_str2(
                     StockMessage::ServerResponse,
                     format!("SMTP {}:{}", domain, port),
-                    err.to_string(),
+                    format!("{}, ({:?})", err.to_string(), err),
                 )
                 .await;
 

src/sql.rs

@@ -49,7 +49,7 @@ pub enum Error {
 pub type Result<T> = std::result::Result<T, Error>;
 
 /// A wrapper around the underlying Sqlite3 object.
-#[derive(DebugStub)]
+#[derive(Debug)]
 pub struct Sql {
     pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
 }
@@ -1241,6 +1241,15 @@ async fn open(
                 .await?;
             sql.set_raw_config_int(context, "dbversion", 63).await?;
         }
+        if dbversion < 64 {
+            info!(context, "[migration] v64");
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN error TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 64).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)

src/stock.rs

@@ -179,6 +179,12 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
     UnknownSenderForChat = 72,
+
+    #[strum(props(fallback = "Message from %1$s"))]
+    SubjectForNewContact = 73,
+
+    #[strum(props(fallback = "Failed to send message to %1$s."))]
+    FailedSendingTo = 74,
 }
 
 /*

src/test_utils.rs

@@ -40,6 +40,24 @@ pub(crate) async fn dummy_context() -> TestContext {
     test_context().await
 }
 
+pub(crate) async fn configured_offline_context() -> TestContext {
+    configured_offline_context_with_addr("alice@example.org").await
+}
+
+pub(crate) async fn configured_offline_context_with_addr(addr: &str) -> TestContext {
+    let t = dummy_context().await;
+    t.ctx.set_config(Config::Addr, Some(addr)).await.unwrap();
+    t.ctx
+        .set_config(Config::ConfiguredAddr, Some(addr))
+        .await
+        .unwrap();
+    t.ctx
+        .set_config(Config::Configured, Some("1"))
+        .await
+        .unwrap();
+    t
+}
+
 /// Load a pre-generated keypair for alice@example.com from disk.
 ///
 /// This saves CPU cycles by avoiding having to generate a key.

test-data/message/gmail_ndn.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a1c:b4d7:0:0:0:0:0 with SMTP id d206csp3026053wmf;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Received: by 2002:a5d:4651:: with SMTP id j17mr19532177wrs.50.1589819005555;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1589819005; cv=none;
+        d=google.com; s=arc-20160816;
+        b=IZbNnzzuYzTFuqfuZwpd3ehqpYYGpn31c8DsfGbQ8rpbS0OTTROkVYvihQl8Ne/8X/
+         brEWsrcmaCh9WpFMzpI+cp/TY39uusnI6qdp5rcgrFmFgoANtwf3TBBj1+f7wBPn46BP
+         dQOUsg/J8KVfvzVgvL1x4uyJ0m9QirDgJeJ/BvrswbTleRQK7oY3fIireUCDxj6r2lCB
+         1Z0TKw1mgIb1LiFMZz8kvCNn3R4KSFnwS8rIju0hYwnsioNiExVQgumXL+RVkEZ9BMzf
+         UdoWIAw3VW+MOZFTpfLCEfgIPtLg/gtE0Q1P+a3KKpi8dkPiV2n6DGMecy9lTLtdhCXt
+         pnaA==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=nNP0DktrSjdBaFfhhoDi2O9KVKM0iXE5ZgubQ0q0ff68Z6Ke7c8dDBXEsZoToI0s4Y
+         w90KyJFpgMJLFmP3iVDRqCfohi2y1HGdWg5VXQPTvzM7+YozZRlbNNV9UsuyRY91CXrJ
+         a2XREBgB+LPMGQivwcHtUMZfyNv/4uiwWivk+92ySNDhxqOiDt4R5Jak/7RkZMFwQpsE
+         JGwk6asM6VqZlihkF24lKv3pPaob6feyX3wD5N0+Mqiy1kQTj2JkpQk6nkTmdf0gapZe
+         fOhU1NkbNfbuS3U7m2gEUiyktE+MhV/MgAzgBhm9bgNt2gQLVWju8rHkPndfv1PDmEkC
+         FsYQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id s18sor5584435wrb.25.2020.05.18.09.23.25
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=dPisws+OwGFyOy0a612XYZgvz5T71GcJRJtU068/Tce8vN/+ggIQtUsZnZtsphe71v
+         2NvfP9ULxR4cXvomTvhrYAk19KdxN/S7SeyBbmXv3x/tg+DBVCmmPS/6RXrcl6Ms3Hkw
+         uPFQ9S3KcvHe/2bcb5LSTA/stIP4tuxxAXvsX2j+MjPYPWKAl50jkSbWK98U0Q0U+MTl
+         pKaaC9s9iEBafac8BFZCy4DfpumKlemNEyRa3cSV2hw+DYHKA5peModrK1A2tcsfstFF
+         rZi8yF/D90RIFbE04DI2QCxB3trsChNF1aYF06aSzI//wsfM1+lb+uGPi0YVkw3n4HrX
+         Xw4w==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=A/NCOtgbpA7VzB1G7ZFo8TA2FfrjuqjGdwMrJr3yXe21FrBFwzssprJwOkynqoVLkK
+         iJU7uMF/KTcQPDEmOLFThzFfe5GCx7eJtZPhwY+FbBlC5sq4I55/xaQLd0gOZ1BYXwMn
+         2bk169d2aoukbaLbGSQZF3d9atd+/e48YzkRxpmUoLcrWk2LcHAeQIG7SgT9pfX5DKPr
+         VpxM5/GMVEBbTRhBIWCeVSfpYCs80l0xEeTC3/B5lzpzMVDE8QCW6Dwh75b4Tb2K6yru
+         Zsy5ZpRmwv0wrkrb2vM+pl4IMkaF7s8XosIvlIT++fQV5xDFItT4atpykZvSDB92RKV0
+         8lEA==
+X-Gm-Message-State: AOAM532RG/PT3ChZHBCDORGLtAjKvX8TGBuOy+AxrnEaJT6v1ieb+VV1
+	+ejly+/6UthxHYlkOJYAszCSgL4dKVFotoVaN7LhEA==
+X-Google-Smtp-Source: ABdhPJz6veVKWhomCL4gK+whrybuMzHCDCq8AowgQvi7sobpMoM/k9CDw79jo1j3OUcTz6MEeUYLxEXuNIuu4zyoS7kVtsUYryGFHAI=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545185wrv.159.1589819005394;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000012d63005a5ee9520"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a5d:5183:: with SMTP id k3mr13704211wrv.159; Mon, 18 May
+ 2020 09:23:25 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ec2b67d.1c69fb81.213af.67a5.GMR@mx.google.com>
+Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+In-Reply-To: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000012d63005a5ee9520
+Content-Type: multipart/related; boundary="00000000000012dc0005a5ee952f"
+
+--00000000000012dc0005a5ee952f
+Content-Type: multipart/alternative; boundary="00000000000012dc0705a5ee9530"
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--00000000000012dc0705a5ee9530--
+--00000000000012dc0005a5ee952f
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--00000000000012dc0005a5ee952f--
+--00000000000012d63005a5ee9520
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Original-Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+Last-Attempt-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+
+--00000000000012d63005a5ee9520
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=a185ogBcMzF9whNVWvuyUoUunNZk3Vc1kEIFmPkX0IxLpAFcI+fOQajOSromGl7Oyi
+         yecLwQevpww2Xd0XjZ3UkZvrI9m9koRmh0QeoHvgTRORiVwj08+PVc3N4F9bCO4w9i0J
+         ir7SSsJqBCDovoIFSFDyNa64vs6Nxno0cH/DaPG7pVTdD+3jfB7nLXIsMQYeX+1eP6rB
+         UhKxH82r7Mh9CI2PWDQpVtGj63AMUEyHgE9Ou08PWbbKjrQOasoG3Tw8tB1GoN1XYssM
+         rxOTgWEoTiduZ35AUH6h+eChOn9OHuI3SPECcVob70Qndayia3dMKfHMO6sEx9J0Wpic
+         29vg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=miGIfL5BgnkD3wQvS34RtGwRRoh+8gJT5sFFfdX/hVyG/dvjXfdwP4yyNWr8ox8iY2
+         BLlahS4y4VGcbG1e2aYjurnWNytGu6utQcZax/uUngJ0bTOwXW1VaIiEZtqd6gTV+8d/
+         rrfQ459+4vXqIoQf0+Oi/U6dWwgJvPPjjRiToWdF3vIJE8R1iTRdZbW4lkgxSADbmskg
+         noT/gWGWblHtR6uuGuKGJ3bkhJKCBnjavKh0LlbWEeFBZfmVNPRvzEFWHjBDdu5wvSL5
+         0QJ+Qn0Orfn5CJuN3xPfzT1S2rI2iYZx37KX9zyMnZEx0ilkTYqCtBPWkrXRYDSXcxYS
+         Y1ag==
+X-Gm-Message-State: AOAM531vhwpXiK8M12286dOJx0Q5fBl9ZaH6BJKts93GoxvPv0xdryP0
+	jg9wYmoP5MUHudsxAMCYDFsCUMVx2PEywyIsaQqklw==
+X-Google-Smtp-Source: ABdhPJxlVJtTODM3pZZSTbbpAAAQRU8XbmuosDF9fgQZmVwxGZSzRWl22o+moppVRU/r8xMAyf0r3+qXwEBe1vZfjZo=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545162wrv.159.1589819005034;
+ Mon, 18 May 2020 09:23:25 -0700 (PDT)
+MIME-Version: 1.0
+From: <alice@gmail.com>
+Date: Mon, 18 May 2020 18:23:39 +0200
+Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+Subject: Kommt sowieso nicht an
+To: assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000000d652a05a5ee95df"
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/plain; charset="UTF-8"
+
+Wollte nur was testen
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">Wollte nur was testen<br></div>
+
+--0000000000000d652a05a5ee95df--
+
+--00000000000012d63005a5ee9520--

test-data/message/gmail_ndn_group.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a02:6629:0:0:0:0:0 with SMTP id k41csp368502jac;
+        Wed, 10 Jun 2020 05:17:57 -0700 (PDT)
+X-Received: by 2002:a6b:1448:: with SMTP id 69mr2898530iou.83.1591791475733;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1591791475; cv=none;
+        d=google.com; s=arc-20160816;
+        b=a0vSKJPbMtGYFnuk1ye/gnnV00Zvva4OOJTMOyfm13xMJD0YAhzGVfa7Z+wn5sQ8dw
+         VAxpmDHCkjp4jol0C1iutiq2Nl0qma819oFPuuoMLLatKQXHpo8Jt+sL3MnwNR7J5bZC
+         1c6Fjk75EIsRWhJd1HCkm44A6UYHxqqsTnzQCaNiHbjsRsvbggxwlMGSrZ4silxqSDvo
+         Pzd/YDLCvsnZNSNIjIckKAwtGmY6sXctZ+JnOTykXAyL32Milfwy1vRL9xm10Q14biTR
+         +qaIQp4E6WE63g1WHvfAjs0Dru7DalUh4GGl/NAwqVhY1gVyRD5E9/nODyHAfxjvaxDD
+         4sMw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=OJbgbrktMKyczw25z/ib7lSdRX80PEK3Myh9fj4q6mDlXmPPv//Gv069znRQ4QbadM
+         HUXZH0WLMZcGyqI6SvGL/noxQ1O8yP0FYJJKTkoX0Gk2hHzfaE3x1scOP/o2FMMQXIFm
+         S4CgGBD6HHzBJYj/rSL3gzqLzx1Id/z5kTeDvH2cn8JJAcCE2q/nhjTyWUb87geoNlDJ
+         A1HRrLHK/0JOyRjHfg2zZCqIvSi1xmpiHStMyL9mfVyrQs98tsPxaOkJHjLplFARoPlr
+         mmmDvsFg7MPvFqkkANzz4JDHidnfKRULCgnrVj1yTU66UagUpQEGjZqz8/99YuU6nt1t
+         81sQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id w14sor16686480iow.23.2020.06.10.05.17.55
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=aO4aNy7CUOk9O4Jnsue/DvMFY6Ph0C34AbpoxJH+mLZpOmt/KYGCGYWgunZgamF15U
+         Vm8JY5yLKGwkTz2m3abDnKNP4fpl6zeZ5fyk5LvXH2Jema0iocHai6pJZBoFGPnonNmd
+         MscTf1sEltbOxwfOmM1BRHX34c1jW0+8Yd2+Nhg2DPvzuq1brOVin6bUV4VX5EeeuNqT
+         ZTewjJVPmO/B5NQhdpG81FO5w4hKSQ/VzZXnap2thMf3gOmnaoR+tbsnOIAiklcLdJ7b
+         57SKUwI041pwSmh9dffs0STl2GvMRSJyGCtBqMnzXgflqoGTcnPflWgR3LXHM/MIA0q8
+         WqRQ==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=iORAzNvXegQ8oSp4RYb/S168muAiBox769seMk49kDBIvXwI+N8P4mUZq/zDi8DmQd
+         +wlLzVzowQq6EofiSpjOJWT9IC/k8otk15PMGtgHE4BGSSeKn7L30d3ocQS93HzYnLmA
+         VBlHBdFTKrsfKhe2+CQyCosTDGRpbkQLuRRyhxChEq8ltvaOHgbu1+eCeb9PsPuh6OxH
+         kvTHJZeA9A+eLOl26pBmqGIWkr7FlYW0wI6YPoEs9WXX5LSFOQs6fm/9l366eIR7IFFI
+         ihX5LrZl/Cf0lwwYX7fqIMgnHy1K+QnKuEb+dRQGqLbxdIEls9bXIF98iPQVkEWzgSZy
+         ip8Q==
+X-Gm-Message-State: AOAM531ahfHE6oS9/nuni8pNf9bwC+DXAcaLV0owBwNCj9kcTPLCCNhX
+	W1JNciK0ivEIVB4dgiyLE/5K7iKbEznQhqyG9Bi1QA==
+X-Google-Smtp-Source: ABdhPJygljUXswH0ycJyHmXVthi5IjlDvP8QdYlMdHUPKEtgIZeUk69Acti5LnswGhg63T9/L0PuGZGBM5XE5BsP0mMNNDRZyt+DgnE=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990163jab.84.1591791475516;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000074432a05a7b9d512"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a05:6638:101c:: with SMTP id r28mr3059870jab.84; Wed, 10 Jun
+ 2020 05:17:55 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ee0cf73.1c69fb81.6888.c2f4.GMR@mx.google.com>
+Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+In-Reply-To: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000074432a05a7b9d512
+Content-Type: multipart/related; boundary="000000000000745e0805a7b9d51b"
+
+--000000000000745e0805a7b9d51b
+Content-Type: multipart/alternative; boundary="000000000000745e1705a7b9d51c"
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--000000000000745e1705a7b9d51c--
+--000000000000745e0805a7b9d51b
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--000000000000745e0805a7b9d51b--
+--00000000000074432a05a7b9d512
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+X-Original-Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+Last-Attempt-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+
+--00000000000074432a05a7b9d512
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=eyr60XbgOrgHoZFpRYzw9WQIR7aEBaYKWhiEcqdnugB+hn0W2KVcTkKiL2C6zSF+jh
+         l+lM+dNZZTUcMqWx4kVgTVtqwUNea8OUqe+WLqx04ULwdKZn1okbKYovaiavCLKOKDnf
+         ZP5mNz3Ka/ywpCGoq8rdgnXc7NunnkWeaBpYY/BWOmLU4WNXX8zS7etXXhQE4YPQEJT4
+         Sh2o/YIIjDLncJFMyE+25n3tbd2mIoLt4sjaCHE5ibm9w7zojyHM+LDCQ37cM74FEAAa
+         88KTn0gSnCFBCfojhfxOH78CpySHG3FFfTlpCefwP2A5J9MQlb6QdSVa9STYSx3IntJ4
+         L7Tg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=pBL4/bKUDw5E2zo1uR2Tl69h2iTlMgIAcnzQgodPCbU4jZ9kH+F5H9rfbzXCjT06J7
+         L72SYpdfgc5fOwM4GhRcdYnyK3wiXQ8ugpL19nbYt2iWo/vRF3GidawXXDGb2GUYpkzX
+         1Mz531cy2/HOsmQbUQ7304KV+OUghtcg8eLNnFuhQch7n12Kk3yy3AOzjrLoktcdgIsy
+         /HxBjyut0Au+A2t6si+PVwTHvC647a0BioeV0tUYLigzu3/jgP9Hb8eRZaXTX5VC6iZi
+         9QMH/+rXp05IK7OpGWh22xDpeV8CDkQ2sLFaBhKxtJ+nYoerM64t8EJXBBsVQb18ojGz
+         pW/A==
+X-Gm-Message-State: AOAM5330q6kn/TKataMNEVigNfNdr/xii/PQgHXzJyMbwLvsETlNfLoy
+	1rM9JBIGrcHeEDRx4qhZfl5S4bircceU7c3i6Fyn2fRO
+X-Google-Smtp-Source: ABdhPJwysG+S90b/g+9mK7LgeHhmJTBowst6JMhL16+a0coTi7P1NVp9jjaNHJfhvhLodYG6eHIvWdbQGJnAP2brEzI=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990137jab.84.1591791475066;
+ Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+MIME-Version: 1.0
+From: Deltachat Test <alice@gmail.com>
+Date: Wed, 10 Jun 2020 14:18:26 +0200
+Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+Subject: test
+To: bob@example.org, assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000006d8d7d05a7b9d5b3"
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/plain; charset="UTF-8"
+
+test
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">test<br></div>
+
+--0000000000006d8d7d05a7b9d5b3--
+
+--00000000000074432a05a7b9d512--

test-data/message/gmx_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Received: from mout-bounce.gmx.net ([212.227.15.44]) by mx-ha.gmx.net
+ (mxgmx101 [212.227.17.5]) with ESMTPS (Nemesis) id 1Mr97m-1jC6Y01o86-00oEqk
+ for <alice@gmx.de>; Tue, 09 Jun 2020 14:35:30 +0200
+Received: from localhost by mout-bounce.gmx.net id 0LhiZF-1jDTj11ZoH-00msO3
+    Tue, 09 Jun 2020 14:35:30 +0200
+Date: Tue, 09 Jun 2020 14:35:30 +0200
+From: "GMX Mailer Daemon" <mailer-daemon@gmx.de>
+To: alice@gmx.de
+Subject: Mail delivery failed: returning message to sender
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+X-UI-Out-Filterresults: unknown:0;V03:K0:O8yx6kuPaGQ=:0wIDPNXEr0wX2oNsLnXaWA
+ ==
+Envelope-To: <alice@gmx.de>
+X-GMX-Antispam: 0 (Mail was not recognized as spam); Detail=V3;
+X-Spam-Flag: NO
+X-UI-Filterresults: notjunk:1;V03:K0:QcE43EBhMmU=:IC5vvzi9jhPS/698Wuubzw1Q4N
+ h87X9j9B3CBN0ZKXB67KepwyNHmh9pxmFIUOMimylv7UK9np+j3X55roOd0nX9BmaaZ3Twvqf
+ UaSsxmyU+cNr6m3+oOb3udJBLe2pJEZDk1cOwACb5NXzYPSaIj4APfGCyvrzIx3FGkNuScNBb
+ tCbbKUJ0GB/VmJLB34XfF6dNN+Iwv9IQ9Yrvw/VXv9vWKsi3qRGGUt3yRw5jUKhQlBY21Pnoq
+ m0LqoMbAKfH1tKEQ/5TymH1ei50YKyWzZ89ISkQwkbYLaqN+6meGACpY18j43VCU9Fk4WQR7y
+ 3XvBYh2CO0CnCn+M9VsnasYag2sNrySe9nzyKfRTaxEg8qlJtl7kS4GX/FsxhHPavkqnU62Gl
+ 9V5TxIG7tmIR0Bf11sPzG/WGegoOHxrfz+qYR81llLMOHznpdDRKjsYDtO/rFBGZzYTiCZsrW
+ dZPVXV25SVcrDGZOaop3JoCbglmXLcSLLhmfE5MzyJEGte3I+6EiZJNeIe8qN3wMDTsRtJL9S
+ J6b2F/5/kTGVAWnXtNlf69BholCrxvjC4Snt3Xjc+7WIO8iw2c5YjmWy+4bAwd4uWll529hZd
+ 6pUYGwjFRnKleivCaJIt7DqbvbE7GZSbQH8fXm3zYqYTrrxiWdlykdoOA1OGbeM06RHJt3mJB
+ osZPU8BZKt0OiBOW64vg6gyAsNC0f02EA7dvRWYgFYqlSogfWZQIOKDKibMVHpIaA0foXg4BG
+ TEQDlsTIL0n2WC9WVqkMdm6xUXHgpArCrAsUhw3mEqPywEfJeBHn60tP2vQ9+pDIQAj5dQCDV
+ y96qSiCX4p31HfrWwAXB9mHfl4OO/tPcKUGBclj2rZ/NMc4O+7yDedLWXQnRzQExfOJLBbBh3
+ xgiNlWFHvDLn0pKG9EI1+3wJ7m2GF2jzDtbQTBv9z26DuAq5WbHZHupzeyfP7VCVXcKuB6sG1
+ 3+LWcdYtcXfqT58HwcvDLwowC4uJpiHfHwtVdiGMtHnmYLysp0V425g+vofQfNzBgR3d9JC15
+ G+HS44o6x6Legm6KnHYH3k0KhR7fgcgswJv/S+I/ryppUhGb2jezVZIUzgvAplzIUDAWnrHdF
+ KVqZ5wBJ0acShIfgMlsIxnBmcnIQ4R4jq3zAyj4XTFxVUFanU8ySiXubxV5PzJqj+GsVa2sjx
+ 9n/xQRJLwgMC4BYqzP6lEPwg/g5AneDAnl7ZlcQPC4SCMblC8N1KZyyIDTXPOI/o4lfdMYb9P
+ 7DmBp2S8aA2yuDe5XT20OmX3kVWeBOsBaAGvVFpIn7gwIDqnFh9WSMP6mkCwfChN3D1yLquYB
+ KAODgRZV5lVNmK+eOjW8m2oiRxmfxrjXLtw5PEhn3RkiRN4HnoePJeoYC7SG4EUwg+wYPu3M6
+ exP/YigoE6bjuBS5d0imUTRDMiwg469GyrFo1J1GkRVvj3lXSF4Nt11j6waqu1l0ReDYfU+QM
+ EMPGLEh1vRChCaqz4L7YI5FlSAVXxfmst0JRyE4k5r9CToEbZuYlPQ+jbcvptwBSaqzMb/YfT
+ cCrU2RWHUfmYIy8x8A/t8ScRbYPzs1lTK3yn1hYeXpw8Fgkip6DIIXAJwUUp+2SLcELICIo+p
+ uumMN0P/OZHH3V/hZ0dPr9xsYi7gdd/vyRIRPUwiL1rSp2WJGi+w4atun8kQBgnbAznRObDh1
+ 4zzKhApX9jo5gtFN6640QDEI5KpsMuPoty4rj9OK163ntKWGR451n+5ZX2FilTlpZYlIPO9Hy
+ SrjHzBog4texceR1OLh4pb/WFB0XFSjQchPAltXCYQFs82aDDk/A0nOPk=
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of
+its recipients. This is a permanent error. The following address(es)
+failed:
+
+snaerituhaeirns@gmail.com:
+SMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please
+ try
+550-5.1.1 double-checking the recipient's email address for typos or
+550-5.1.1 unnecessary spaces. Learn more at
+550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21
+9 - gsmtp
+
+
+
+--- The header of the original message is following. ---
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=gmx.net;
+	s=badeba3b8450; t=1591706130;
+	bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=X-UI-Sender-Class:To:From:Subject:Date;
+	b=NwY6W33mI1bAq6lpr6kbY+LD2hO9cDJBItTgY3NRIT94A6rKTVlSmhFM3AxYgFnj0
+	 Db0hncsNRDqcdtRoOo8Emcah5NJURvEQohG37lkug3GqneB4+FNTdYCeQbOKlZn6on
+	 pYYD/T9CmeL2HG3+8voeBjZIUenyXrF2WXG37hFY=
+X-UI-Sender-Class: 01bb95c1-4bf8-414a-932a-4f6e2808ef9c
+Received: from [192.168.178.30] ([84.57.126.154]) by mail.gmx.com (mrgmx005
+ [212.227.17.190]) with ESMTPSA (Nemesis) id 1MKbkM-1jNoq60HKm-00KyL2 for
+ <snaerituhaeirns@gmail.com>; Tue, 09 Jun 2020 14:35:30 +0200
+To: snaerituhaeirns@gmail.com
+From: Alice <alice@gmx.de>
+Subject: test
+Message-ID: <9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de>
+Date: Tue, 9 Jun 2020 14:36:10 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+X-Provags-ID: V03:K1:7awoptmynaF5MqxCAincdsa7zFCwQFNkFb6xRxigK06uEiGN00b
+ Mv2wJU91CEd4mvCCWzrTtaWLZDLH8pjAWaT4+HvYIUbpwNx2jC6WHTppkYYRMVJnm4lG9pr
+ SUx1OlIcp0kbsnl3mB9xYNFwm9jzpR9Kx8QEHwIbZiiSFBcH56498UGQi//kKXVMos8C14o
+ I7cmwYmr8xB09DwLMKXfg==
+X-Spam-Flag: NO
+X-UI-Out-Filterresults: notjunk:1;V03:K0:NWQAUbhAkBc=:yAaolOnVCDWEZhgUwwvtEs
+ wXbSJ/GfMvDRpCkYpFBvHXOTpGm6hjdjQ0vLK2hvu/Hz22UdlWbIdc1J2oO9S5U20mIdc+1bS
+ TPSSpqPFc7ICPx4Wbvv2SEp9ZqH2q7ORC52UvUWfI6OjAJEPDNrXQFdUiZAa72hLj1NPeG6Qi
+ 4AbL2HwLfJ8s6TeOCm6TXRRuD+w1o/ASFOqQmoao2dFyZ2BaoAgOKPKxXYfwVGceuUygpchyS
+ 0d2bZYOXSLR+6rUYevjZAq1OCi9AIC6/wlkOe5yIRk4gJFMfPauaICsdnq3uZ9ikCAX83VWun
+ PJVMxTLTP54lgo2h0jMBX3uKk10+/wzXWplllxX9NnSa3x1V28n6raslNF0IoC6Pm72kC6Jzr
+ GkC22viCm3/Y4uHlPMOXbY5WFrQe/D9GKeJeXBLoGciNwIFkUG12a+iqWtoT+h5HVObTW8LxM
+ +UtEl97nAwxYSM+sGfIpasRpZc7r/SgN3JWGO9R9WaXpW4Cc1dH7RI+hzuZDsDUBEGTUTVPDo
+ 0SvjKHiJ6sUqGDyfv4HUgVutus6EYP27LALND4ekfom2DPRFopZhbtV5fZT7CL1Q1NogU7tYf
+ /FdmR1T1J1zCAZSFvyR5LBkfglZlHzgdnTF2heuxyqKq4dm0hnLFSULB1+CVWsg8hzrruvO5q
+ XzA6qIhBQUZmWo7wBpqpkBPxzjgTGGtXc5y5e6+crxYbbuQdnUWEnyw2xI4d6pJPqtHDA2/vT
+ ZgvNDUGceavTR5Rtyb14hhX4Q6dWK3ATy16j4hs9Aq+q/IKyVAX3A5nFYbJRIz+2YnoLr2YOa
+ IrScEorXjvTxjw+aBy73SZBe2REPzJ+O6k7chVrYjV9Q28FiGVuRYJYxWw/59Pes7IAbmfQBV
+ 4vqGCQQr4eG78gVwjw+SNdp4/6jdNkIHDqR4XW9id/r5wYxKKj4UUkSor3/+h9Rd9srh+GApy
+ uOxw/ejFvbRcxFIjvadpq1KLnO7nM27nJ4lp44ul3i7VUGefLM/45TCsuds2HM1iQWhPFQ54y
+ SA5sYjf73EUJdkHchaf5i+4uSOmbOWQ4Yvmd8+IoyoXAxvEzY2Xh53nWi8ZPY1Tu4Bw8GRrz7
+ L+VK0QiWCg3/hM7wRlFFyshmMrScMk5fOf9ynqd0JbHB7u+n4/GUwx3im/w8+NgSd3YOz7wNU
+ KD1snDWoMUO8f23Ik1Osym688OLWNwKYT+mZbMIMXcz1fB+olRZn4czMhN5DiSb8hyOxRI8NE
+ PNfaoN87CXiRkgazV6U1eiRkfcK2AvI7zOJF1tclUHZ9awyYoXtxfEzZ+J/2TCXiC7V2iSkUF
+ EjwgPxlJmccccjsxc46v1ajnTxLo0tJbZ0+DJXWkCgQ0d/iiScQ=
+
+

test-data/message/posteo_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Delivered-To: alice@posteo.org
+Received: from proxy02.posteo.name ([127.0.0.1])
+	by dovecot03.posteo.local (Dovecot) with LMTP id zvCFJRzX317LGQIA+3EWog
+	for <alice@posteo.org>; Tue, 09 Jun 2020 20:44:24 +0200
+Received: from proxy02.posteo.de ([127.0.0.1])
+	by proxy02.posteo.name (Dovecot) with LMTP id mhNkNAnR316xBQMAGFAyLg
+	; Tue, 09 Jun 2020 20:44:23 +0200
+Received: from mailin06.posteo.de (unknown [10.0.1.6])
+	by proxy02.posteo.de (Postfix) with ESMTPS id 49hJtv3RRcz11m7
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+Received: from mx04.posteo.de (mailin06.posteo.de [127.0.0.1])
+	by mailin06.posteo.de (Postfix) with ESMTPS id 6935920DD2
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+X-Virus-Scanned: amavisd-new at posteo.de
+X-Spam-Flag: NO
+X-Spam-Score: -1
+X-Spam-Level: 
+X-Spam-Status: No, score=-1 tagged_above=-1000 required=8
+	tests=[ALL_TRUSTED=-1] autolearn=disabled
+Received: from mout01.posteo.de (mout01.posteo.de [185.67.36.65])
+	by mx04.posteo.de (Postfix) with ESMTPS id 49hJtv001Vz10kT
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Authentication-Results: mx04.posteo.de; dmarc=none (p=none dis=none) header.from=mout01.posteo.de
+Received: by mout01.posteo.de (Postfix)
+	id DCB6B1200DD; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+From: MAILER-DAEMON@mout01.posteo.de (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@posteo.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="B39111200B9.1591728262/mout01.posteo.de"
+Content-Transfer-Encoding: 7bit
+Message-Id: <20200609184422.DCB6B1200DD@mout01.posteo.de>
+
+This is a MIME-encapsulated message.
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Notification
+Content-Type: text/plain; charset=us-ascii
+
+This is the mail system at host mout01.posteo.de.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550
+    Requested action not taken: mailbox unavailable (in reply to RCPT TO
+    command)
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mout01.posteo.de
+X-Postfix-Queue-ID: B39111200B9
+X-Postfix-Sender: rfc822; alice@posteo.org
+Arrival-Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+
+Final-Recipient: rfc822; hanerthaertidiuea@gmx.de
+Original-Recipient: rfc822;hanerthaertidiuea@gmx.de
+Action: failed
+Status: 5.0.0
+Remote-MTA: dns; mx01.emig.gmx.net
+Diagnostic-Code: smtp; 550 Requested action not taken: mailbox unavailable
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Undelivered Message Headers
+Content-Type: text/rfc822-headers
+
+Return-Path: <alice@posteo.org>
+Received: from mout01.posteo.de (unknown [10.0.0.65])
+	by mout01.posteo.de (Postfix) with ESMTPS id B39111200B9
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Received: from submission-encrypt01.posteo.de (unknown [10.0.0.75])
+	by mout01.posteo.de (Postfix) with ESMTPS id 8A684160060
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=posteo.de; s=2017;
+	t=1591728262; bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=To:From:Subject:Date:From;
+	b=brJnt4PLAX3Tda1RHCo91aB1kMAL/Ku9dmO7D2DD41Zu5ShNsyqqyDkyxb1DsDn3O
+	 6KuBZe3/8gemBuCJ/mxzwd9v8sBnlrV+5afIk0Ye9VvthZsc4HoG79+FiVOi9F38o0
+	 DtJJFYFw/X7mAc5Xyt0B0JvtiTPpBdRAkluUQm+QW6cW6GGlwicVW19qvebzq+sHyP
+	 X2bZ8wpo78yVgvjPBK3DLaXa+pKFMBjLdDUcIE2bZnY6u6F1x8SXGKGBoxVwdJipJx
+	 v14so5IejNsf4LYJjH3Qb8xgK1aAi6e6nQn4YXV0INL6ahzgALiT9N6vwunNKYVJNi
+	 fPPKvBWDfUS4Q==
+Received: from customer (localhost [127.0.0.1])
+	by submission (posteo.de) with ESMTPSA id 49hJtt1WPbz6tmV
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+To: hanerthaertidiuea@gmx.de
+From: deltachat <alice@posteo.org>
+Subject: test
+Message-ID: <04422840-f884-3e37-5778-8192fe22d8e1@posteo.de>
+Date: Tue, 9 Jun 2020 20:45:02 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+Posteo-User: alice@posteo.org
+Posteo-Dkim: ok
+
+--B39111200B9.1591728262/mout01.posteo.de--

test-data/message/testrun_ndn.eml

@@ -0,0 +1,107 @@
+Return-Path: <>
+Delivered-To: alice@testrun.org
+Received: from hq5.merlinux.eu
+	by hq5.merlinux.eu (Dovecot) with LMTP id Ye02K6PB5F43cQAAPzvFDg
+	for <alice@testrun.org>; Sat, 13 Jun 2020 14:08:03 +0200
+Received: by hq5.merlinux.eu (Postfix)
+	id 9EBE627A0B2E; Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+Date: Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+From: MAILER-DAEMON@hq5.merlinux.eu (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@testrun.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="CDB8D27A0B2C.1592050083/hq5.merlinux.eu"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20200613120803.9EBE627A0B2E@hq5.merlinux.eu>
+
+This is a MIME-encapsulated message.
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host hq5.merlinux.eu.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1
+    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in
+    virtual mailbox table (in reply to RCPT TO command)
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Delivery report
+Content-Type: message/global-delivery-status
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; hq5.merlinux.eu
+X-Postfix-Queue-ID: CDB8D27A0B2C
+X-Postfix-Sender: rfc822; alice@testrun.org
+Arrival-Date: Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+
+Final-Recipient: rfc822; hcksocnsofoejx@five.chat
+Original-Recipient: rfc822;hcksocnsofoejx@five.chat
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; mail.five.chat
+Diagnostic-Code: smtp; 550 5.1.1 <hcksocnsofoejx@five.chat>: Recipient address
+    rejected: User unknown in virtual mailbox table
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Undelivered Message
+Content-Type: message/global
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <alice@testrun.org>
+Received: from localhost (p200300edb723070079835ce22985a199.dip0.t-ipconnect.de [IPv6:2003:ed:b723:700:7983:5ce2:2985:a199])
+	by hq5.merlinux.eu (Postfix) with UTF8SMTPSA id CDB8D27A0B2C;
+	Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org;
+	s=testrun; t=1592050082;
+	bh=Kvhta0OMsTRVC7OlaAqo68TBE0KuGBv4vUBp6Ez/7VY=;
+	h=Subject:References:In-Reply-To:Date:To:From:From;
+	b=Ql60JEGFXLNvjsyihATw2z34ct++8xZvTPNw0snXe6+oqdqsRZJ9tWNDTxOgx8Iqf
+	 HQ4puBVGcWjIlszYQVLlq3APi04o2ep3GrD8EF0J0GpDdW8yw6wCos6Q8r+TWmXwET
+	 kGXHTRPVaUIqZF2i/utypxMfd1ua0S3jBDnIXTe/p2+XvfC3Cf3hZGW+FQ/Zd7G8Vh
+	 /U2rgX5BTIGf26ZCbmcMaXWkftgv6+yns0AmzorV9yB+EhTkWIUjk+C25bRtMbJ5mZ
+	 93dwdr+sXrrSZLSi+LBqc57Dv9j4p/SUmB4zPlvfUv7/bqLi36pypvtCJ5Ul8UEXSb
+	 XNFZPaEl+mwjA==
+Content-Type: text/plain; charset=utf-8
+Chat-Disposition-Notification-To: alice@testrun.org
+Subject: =?utf-8?q?Message_from_hocuri1=40testrun=2Eorg?=
+MIME-Version: 1.0
+References: <Mr.VSg3KXFUOTG.9sn7JBxZn1W@testrun.org>
+	<Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+In-Reply-To: <Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+Date: Sat, 13 Jun 2020 12:08:01 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@testrun.org; prefer-encrypt=mutual;
+	keydata=xjMEXt3z1xYJKwYBBAHaRw8BAQdAf6MctU/8cmEqwEN9VFZ3gHBFIxKiEaARZl1DFUkI7e
+	rNFTxob2N1cmkxQHRlc3RydW4ub3JnPsKLBBAWCAAzAhkBBQJe3fPXAhsDBAsJCAcGFQgJCgsCAxYC
+	ARYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygAG2IA/1nTmmmkHAc1Bjtx2FOstbaS+N
+	XHjxaK+hkoWllsyhz0AQDJJ1++u7jVZPRn/j1LlByrT3Jv/D1aY14J5rjj+ADVBM44BF7d89cSCisG
+	AQQBl1UBBQEBB0DpSTaZ30dAVwM9PkBe2h+gFyxn9HSorP4XCHJu/lIdPAMBCAfCeAQYFggAIAUCXt
+	3z1wIbDBYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygA2QkA/16toWCtseYKw8G1X2j7
+	xYR3Cyabq37hgbesDOThIIzNAP0UCUS8mnunmkS5adEbftRaDi2JZoGxDw46jtJJ2+13Cw==
+Message-ID: <Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org>
+To: <hcksocnsofoejx@five.chat>
+From:  <alice@testrun.org>
+
+F
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu--

test-data/message/tiscali_ndn.eml

@@ -0,0 +1,91 @@
+Return-Path: <>
+Delivered-To: alice@tiscali.it
+Received: from director-5.mail.tiscali.sys ([10.39.80.174])
+	by dovecot-08.mail.tiscali.sys with LMTP id SBRfEpGb517VAgAAd2fHbg
+	for <alice@tiscali.it>; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from cmgw-4.mail.tiscali.it ([10.39.80.174])
+	by director-5.mail.tiscali.sys with LMTP id MFUPL5Cb516tawAArQJVuQ
+	; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from michael.mail.tiscali.it ([213.205.33.246])
+	by cmgw-4.mail.tiscali.it with 
+	id rTtS2200V5JdeUd01U2RlV; Mon, 15 Jun 2020 16:02:25 +0000
+x-cnfs-analysis: v=2.3 cv=ZdPMyfdA c=1 sm=1 tr=0 cx=a_idp_d
+ a=AfTPebshMYb+aQOCLa9q3Q==:117 a=HpEJnUlJZJkA:10 a=jmdcTMp_Gj4A:10
+ a=r77TgQKjGQsHNAKrUKIA:9 a=b8iBRs35AAAA:8 a=NMdB-582e605uxHDr_AA:9
+ a=QEXdDO2ut3YA:10 a=MhhPCb74-dYA:10 a=HXlsH_Kov2KnitTn7A4A:9
+ a=BkuCPOF3BOzethIN9HQA:9 a=qG5HpJ6ZyD35YNEB:21 a=kvHihYffoorsyJbA:21
+ a=xD8EQi6zkreDqSNPYj5l:22
+Date: Mon, 15 Jun 2020 16:02:25 +0000
+From: Mail Delivery System <mail-daemon@smtp.tiscali.it>
+To: alice@tiscali.it
+Subject: Delivery status notification
+MIME-Version: 1.0
+Content-Type: multipart/report; boundary="------------I305M09060309060P_896715922369450"
+
+This is a multi-part message in MIME format.
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/plain; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+      This is an automatically generated Delivery Status Notification.      
+
+Delivery to the following recipients was aborted after 2 second(s):
+
+  * shenauithz@testrun.org
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: message/delivery-status; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; michael.mail.tiscali.it [213.205.33.13]
+Received-From-MTA: dns; localhost [146.241.100.150]
+Arrival-Date: Mon, 15 Jun 2020 16:02:23 +0000
+
+
+Final-recipient: rfc822; shenauithz@testrun.org
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550 5.1.1 <shenauithz@testrun.org>: Recipient address rejected: User unknown in virtual mailbox table
+Last-attempt-Date: Mon, 15 Jun 2020 16:02:25 +0000
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/rfc822-headers; Content-Transfer-Encoding: 8bit
+Content-Disposition: attachment
+
+x-auth-user: alice@tiscali.it
+Chat-Disposition-Notification-To: alice@tiscali.it
+Chat-User-Avatar: avatar.jpg
+Subject: =?utf-8?q?Message_from_=F0=9F=8F=9E=EF=B8=8F_Mefiscali?=
+MIME-Version: 1.0
+Date: Mon, 15 Jun 2020 16:02:22 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@tiscali.it; prefer-encrypt=mutual;
+	keydata=xjMEXtFRUBYJKwYBBAHaRw8BAQdA5sqHJqkWlveCgsNd0rtwtZrT1mmo1gwaGC5+WheYk5
+	nNHTxhbmRyZWFzLmxhdHRtYW5uQHRpc2NhbGkuaXQ+wosEEBYIADMCGQEFAl7RUXoCGwMECwkIBwYV
+	CAkKCwIDFgIBFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrAAaAEA4wssXeU2IXnowv
+	iu3zmcNzDgE4HdmW4RFyqJC6bgxXQA/3aTfE/PhQgZvi6RrKMvP4zygXpD9y+3ydIZP88Bp8kIzjgE
+	XtFRUBIKKwYBBAGXVQEFAQEHQKaAwlP0j9m0aYsCtO+qD9+foH0kiTN5BWDe5YcZrckVAwEIB8J4BB
+	gWCAAgBQJe0VF6AhsMFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrA1JgEAkscCQlps
+	h3ZxlLqBlbf2+85f4S4aGQfFPtIYEkKKhYEBAJbQulNNp9UarvhfyBiIdvkBVDcCnJZwzbORqp8RM0 gC
+Message-ID: <Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it>
+To: <shenauithz@testrun.org>
+From: =?utf-8?b?8J+Pnu+4jyBNZWZpc2NhbGk=?= <alice@tiscali.it>
+Content-Type: multipart/mixed; boundary="5uAmYQux1HZxxriijTjjKSp4DMoJwq"
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tiscali.it; s=smtp;
+	t=1592236943; bh=C3taz+zuSre1ko5Q5CzGPmbyrgegKYBClx/3Dv7t/Xw=;
+	h=Subject:Date:To:From;
+	b=LrcfLfrQoemOkHTQsqR8MExqNlx5KPYNFWhwlBWylvVc5GlmlhzqM6SAVKd0NVsKE
+	 gVRlBId5FvnlwoJ2WZnXaw/+3lWKilMTuzzQ1oFGvLnZ1XUaUEfuliIv+9NI79dJWX
+	 +S3jsSgzJMJc9+fO6s9bJsX1EHQ2a8GXNbwDtLXs=
+
+
+
+--------------I305M09060309060P_896715922369450--
+
+

test-data/message/yahoo_ndn.eml

@@ -0,0 +1,100 @@
+X-Atlas-Received: from 10.218.250.153 by atlas222.free.mail.ne1.yahoo.com with http; Wed, 10 Jun 2020 10:11:23 +0000
+Return-Path: <>
+Received: from 77.238.177.145 (EHLO sonic314-19.consmr.mail.ir2.yahoo.com)
+ by atlas222.free.mail.ne1.yahoo.com with SMTPs; Wed, 10 Jun 2020 10:11:23 +0000
+X-Originating-Ip: [77.238.177.145]
+Received-SPF: none (domain of sonic314-19.consmr.mail.ir2.yahoo.com does not designate permitted sender hosts)
+Authentication-Results: atlas222.free.mail.ne1.yahoo.com;
+ dkim=pass header.i=@yahoo.com header.s=@bounce;
+ spf=none smtp.mailfrom=sonic314-19.consmr.mail.ir2.yahoo.com;
+ dmarc=success(p=REJECT) header.from=yahoo.com;
+X-Apparently-To: alice@yahoo.com; Wed, 10 Jun 2020 10:11:23 +0000
+X-YMailISG: RT5ZnycWLDvIW52uqHS_EWNgl31NdJPyLLB2F4SYb1GCAoo9
+ pcninuVU5GDMBZykeMT4cSUt4ZqXxS5FdEeWJqtGIAtbEGbIL8Uhcoszqm4m
+ JuMJiQZwEE7W_fsS_9MUK5gZtMkhKkSnAuaeaOLKNYAwFZdBqA0uEYA5EmVf
+ EC9J4RGQ4hZvrMqMj_W.cj4pvbEC.pyirLxTfkICuUkZVguYoxG16y1EOJPw
+ B48fhXvF5ErU7WAHKxyRM3bMOg7b5pXHKn1dtRSVAXEuqBAQrWig1pePpYH1
+ wO54sYT7cgmdiFvfLY5rR7YcBzopmKJBycKzBVoRLCY4gvoNyTLPKx9o3AAz
+ WU4B7TGejDBElYSLpfnyvQg8wU27zzo2IVBZWUNztP0Ca8CQ07Y7TxUZAO.f
+ DNO5c7nd81PHMRDbSeaw1BTV2Yd9vlBc7syYmwGvtVBJQwRU7qPN.DpFO2jC
+ 9j9DytVhm5231gdBBRSzW78yG.VvaIdJgq_YViKNM9VxFseTz3Sjt3TaYznP
+ gAVq.MxpopNsSZf_tedwAhXDWyrjKsRPK.v2ANivmuWGPednniEaMYhxJ05M
+ _5SnJ.hAU.l6h3HCEfiU.SH390_3tZgYNfxCo4GPPFMfnNPmKa3.rgpChBCz
+ 9CRexJ8BSFyCEeAhuqQ8vSJfSuittJmXvS6Tk8Rxd9HUJAtKzZ.xCWZQ4tA6
+ Yp2aRG23_rK_C6hH8ArkkvbG.uVQTt6DltSX6avJLObBfIhBH0x64RoFjGee
+ vYXxM741Okm0jH7r79c8GhnAwas_bwfkaTW9e1nhYP0eyI36z_QwLYgOH3Mm
+ LrUcejpOMDR60QWDuDyRbWXOJdr3Q2K0ERhuAy6YnINq0sL3HX7t5wjsFLvp
+ _7Ri_eruTfIst4C7DZwERwui6aDSEAdF1Z8oZukBVmiyZsHmhJQCUik646iy
+ 3ASMR3lX7R3q2PBHQo2oC3qte8Fzz1FhKoMtfCGtIpeCazlkhDEJ6eTBSQ3R
+ Pe7M_GPiv3QNp7qu5CWHlzy6hWEKIkNwx.WRGYzfxkyJMmJm4UrhQYUfa4lG
+ Wb8n.mfYnS_KGYtzyRFNqAL0IGo.1MB9aG6qQk456Fz9GJgbHLWrMXVtyfrr
+ Uo7mKih8FCrdUKv5X6KBnpY0vvyoH5jrWyrvo3DW0bq_JvZ9U51JwUhoGY5U
+ c1t.yCSJbs8tnrGZHuUTOvouWzpCAJsk34AqRyH0wDJZQsAwBW5UZ3jx8ARA
+ FicoSqZCa4wEP9WaaXvfzFbmLW0-
+X-Originating-IP: [77.238.177.145]
+Received: from 10.217.135.165  (EHLO sonic314-19.consmr.mail.ir2.yahoo.com) (77.238.177.145)
+  by mta4277.mail.ne1.yahoo.com with SMTPS; Wed, 10 Jun 2020 10:11:21 +0000
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=bounce; t=1591783879; bh=4BmZzBC/nu0AJ9r0i0xNuCENks2KZcuXCbjSHdzbg9Y=; h=Date:From:To:Subject:From:Subject; b=lPxu8goOGOLVgnwbndfdptZ7zI5VEo0lSSr+ONGxwdtuhrySKDU6Sp41/g6jWbAiVPT1947j/B5wOlPfa5tv4XkWrGf0JCbT1I20ZJIkNfNwt4F0qPnbJAiHFIDPxcY68utjC9IgPWJd0cGqJNXbFwbJBu88rtrbMoInzLakh5I=
+Received: from sonic.gate.mail.ne1.yahoo.com by sonic314.consmr.mail.ir2.yahoo.com with HTTP; Wed, 10 Jun 2020 10:11:19 +0000
+Date: Wed, 10 Jun 2020 10:11:19 +0000
+From: MAILER-DAEMON@yahoo.com
+To: alice@yahoo.com
+Message-ID: <1713051795.39992.1591783879940@sonic314.consmr.mail.ir2.yahoo.com>
+Subject: Failure Notice
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+Content-Length: 3347
+
+Sorry, we were unable to deliver your message to the following address.
+
+<haeclirth.sinoenrat@yahoo.com>:
+554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com
+
+--- Below this line is a copy of the message.
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=s2048; t=1591783876; bh=kXD6TZuQDjqZf/AEAJ2HCX32Titkj3IywytG6GSm4yQ=; h=Date:From:To:Subject:References:From:Subject; b=PexnIBxnVSkyutv/jVn+Wlt5QPuVHnWleP3baWlvqkXaKR51pCaZIoGJggMEonitEeJkcYpgplBawEBz0hlGf63NqOHHkxUk6U1olwc0y9kj5kDH5lrORgXrf7U4z5t+i5n0II36MxbG9n/5tDRXoiabLFoWx//3O9x/ZJvVWPlq8RBFzVG8aoL2TkBQAVcqX/vW1f3WJuopaUYWB4AzR3TyuC2kVQPFqbPMk+G/VmyuZFmZesh1bSBva5hdKYLxES5v8hvTIDRqSYZrnZ4V67MqicZ8m229Xf4Za8qOE+a2Z+Vv5VrQ+CjjPZwRAcmKHkLY80VCSkpeL2R2YG4APg==
+X-YMail-OSG: 4spnw98VM1k_g51CM6oepdNEMiPFRtZ0ZG_zOBGIlhcNvS1mkzr8l8VTB0CY_Q0
+ dR52ikl7QVYESerRQgcGqBfLwKpem7i1XSrCl2HuyHeWzF6Gu5MqFPMCak.v8GyDXNO075NEwNt1
+ i18CJ29cEjiHthoamgmj0oqerAglglKRhTuuAFy4wUmZZm7VyvaW4wHUD1g7DeWGijQsCglSYMUK
+ CmoFcKsWOZBSYPMkp7iRwUp52pXHFin3qf4uQ27K_Sh.6s7KLAfWVkV7L_5AR3MyCPAVzm71.1yG
+ G7Vy5HSBgGMQ90B7VbcjOkCg3F4JNl4Z_P2ejV1KZ.tNoPLgO.FmsfFy1OXBGf3m2mDmRcuEO4K2
+ mTRhsjZf.2iiWpx02b3tY.oUtYrIBXBVIFPbTB9sBMn_9Z_qdVmO3gjD6gCPEBzuVvEO0eZIrgaw
+ EDTZt8Z9tSRDm1.4gV8LWQBYShF7XuMV0togiLYIO8s_iTHcTbhKhPlwxP.mxr06xcx_9kzReVTL
+ 9lB1FkB5Jm0WccWHGhLBqeMjGDoaNqPxLqJ.1tI58tLXsPoR6m1NFVEdzI1G.4AVBeXZ_9BjgUhm
+ KY33sEg.GwIjUlWWWuSyRZ1q1K2nqi1z29wH2R1Glmdmx0lyqfMg9Xe8HV7YZu2CuZ8SlDLLB.rX
+ NU4PwMsNfU6pK2HejQPsJuyOlI5Q824rXRF5xTLYKsYcQptoFXyLe6MXKW1ThBLQV2nWYDRs_V.e
+ SBmt22TfuOwu4Y5ju0sXmztZ8zpiIC8_rnAa5bVBEHxzkic64UZdukDX9V12Pk3G2sGYRyPTH472
+ wBX33JpDuq6BtrKr4FXjCLeVppARTHpiKM0jHMjmNf1bF0TvrcCsC9zAYtitAqgcGZNFETNuV3KM
+ 57XifdDEwUPOuww0ApSWO.iwP2POvIRBVlrxdgA8MbLmuuX4UxNCw23z1f7MVY6F3L60LUrX5GZO
+ aKaMmD1XTzx32J6c_TUmyuViT5vphqpEooTzHG2X7ALb4xC8yHlE4wDKyaEDARZ.8P2lO9T18oCz
+ OQvJjwDaLOkeAmo23yRMn70bYJK3tP9Z5cS1C0TE8PEtz4sd1syQUIZZ2g8JG_AQcE4lUZSZlIKN
+ AHjB8h8Uin35zKe0Le1DBjdQUmpgAETlmYE7V0nJDEmagB3dtpbokgRBuuBfhXlFpxHcnAmBFFFm
+ XOSLWEPnmxu2o8CCjjz3QUBy2fr3EI_D2VFpy..MuZgRwtES.l24m_95xtQxI28R4SWZN6LsS_rr
+ 1S33BJCCCAfXtCAFzCfz5.qSzHRYbLdY5do6yKj0pPLQTUTjlMwmCUGPcSJhsyxkkEVIK1W_Z16R
+ ZRls-
+Received: from sonic.gate.mail.ne1.yahoo.com by sonic314.consmr.mail.ir2.yahoo.com with HTTP; Wed, 10 Jun 2020 10:11:16 +0000
+Date: Wed, 10 Jun 2020 10:11:12 +0000 (UTC)
+From: Delta Chat Test <alice@yahoo.com>
+To: "haeclirth.sinoenrat@yahoo.com" <haeclirth.sinoenrat@yahoo.com>
+Message-ID: <1680295672.3657931.1591783872936@mail.yahoo.com>
+Subject: test
+MIME-Version: 1.0
+Content-Type: multipart/alternative; 
+	boundary="----=_Part_3657930_145367320.1591783872935"
+References: <1680295672.3657931.1591783872936.ref@mail.yahoo.com>
+X-Mailer: WebService/1.1.16072 YMailNorrin Mozilla/5.0 (X11; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0
+Content-Length: 494
+
+------=_Part_3657930_145367320.1591783872935
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+test
+
+------=_Part_3657930_145367320.1591783872935
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+<html><head></head><body><div class="yahoo-style-wrap" style="font-family:Helvetica Neue, Helvetica, Arial, sans-serif;font-size:16px;"><div dir="ltr" data-setdir="false">test<br></div></div></body></html>
+------=_Part_3657930_145367320.1591783872935--