bump version to 1.45

Update async-imap to fix Oauth2

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.43.1
+          toolchain: 1.45.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,21 +32,34 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.43.1
+            toolchain: 1.45.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all
 
 
   build_and_test:
     name: Build and test
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly, 1.43.1]
+        rust: [1.45.0]
+        experimental: [false]
+#        include:
+#          - os: ubuntu-latest
+#            rust: nightly
+#            experimental: true
+#          - os: windows-latest
+#            rust: nightly
+#            experimental: true
+#          - os: macOS-latest
+#            rust: nightly
+#            experimental: true
 
     steps:
     - uses: actions/checkout@master
@@ -79,11 +92,10 @@ jobs:
       uses: actions-rs/cargo@v1
       with:
         command:  check
-        args: --workspace --all --bins --examples --tests
+        args: --all --bins --examples --tests --features repl
 
     - name: tests
       uses: actions-rs/cargo@v1
       with:
         command: test
-        args: --workspace
-          
+        args: --all

ASYNC-API-TODO.txt

@@ -1,54 +0,0 @@
-
-Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
-
-- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
-
-APIs:
-    dc_context_new                 # opens the database
-    dc_open                             # FFI only
-      -> drop it and move parameters to dc_context_new()
-
-    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
-    dc_imex NEVER goes through the job system
-    dc_imex import_backup needs to ensure dc_stop_jobs()
-
-    dc_start_io                # start smtp/imap and job handling subsystems
-    dc_stop_io                 # stop smtp/imap and job handling subsystems
-    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
-
-    dc_close                            # FFI only
-      -> can be dropped
-    dc_context_unref
-
-    for ios share-extension:
-    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
-        0: message was sent
-        1: message failed to go out, is queued as a job to be retried later
-        2: message permanently failed?
-
-EVENT handling:
-    start a callback thread and call get_next_event() which is BLOCKING
-    it's fine to start this callback thread later, it will see all events.
-    Note that the core infinitely fills the internal queue if you never drain it.
-
-    FFI-get_next_event() returns NULL if the context is unrefed already?
-
-   sidenote: how python's callback thread does it currently:
-         CB-thread runs this while loop:
-         while not QUITFLAG:
-             ev = context.get_next_event( )
-             ...
-         So in order to shutdown properly one has to set QUITFLAG
-         before calling dc_stop_jobs() and dc_context_unref
-
-    event API:
-        get_data1_int
-        get_data2_int
-        get_data3_str
-
-
-- userdata likely only used for the callbacks, likely can be dropped, needs verification
-
-
-- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
-

CHANGELOG.md

@@ -1,5 +1,173 @@
 # Changelog 
 
+## 1.45.0
+
+- add `dc_accounts_t` account manager object and related api functions #1784
+
+- add capability to import backups as .tar files,
+  which will become the default in a subsequent release #1749
+
+- try various server domains on configuration #1780 #1838
+
+- recognize .tgs files as stickers #1826
+
+- remove X-Mailer debug header #1819
+
+- improve guessing message types from extension #1818
+
+- fix showing unprotected subjects in encrypted messages #1822
+
+- fix threading in interaction with non-delta-clients #1843
+
+- fix handling if encryption degrades #1829
+
+- fix webrtc-servers names set by the user #1831
+
+- update provider database #1828
+
+- update async-imap to fix Oauth2 #1837
+
+- optimize jpeg assets with trimage #1840
+
+- add tests and documentations #1809 #1820
+
+
+## 1.44.0
+
+- fix peerstate issues #1800 #1805
+
+- fix a crash related to muted chats #1803
+
+- fix incorrect dimensions sometimes reported for images #1806
+
+- fixed `dc_chat_get_remaining_mute_duration` function #1807
+
+- handle empty tags (e.g. `<br/>`) in HTML mails #1810
+
+- always translate the message about disappearing messages timer change #1813
+
+- improve footer detection in plain text email #1812
+
+- update device chat icon to fix warnings in iOS logs #1802
+
+- fix deletion of multiple messages #1795
+
+
+## 1.43.0
+
+- improve using own jitsi-servers #1785
+
+- fix smtp-timeout tweaks for larger mails #1797
+
+- more bug fixes and updates #1794 #1792 #1789 #1787
+
+
+## 1.42.0
+
+- new qr-code type `DC_QR_WEBRTC` #1779
+
+- new `dc_chatlist_get_summary2()` api #1771
+
+- tweak smtp-timeout for larger mails #1782
+
+- optimize read-receipts #1765
+
+- Allow http scheme for DCACCOUNT URLs #1770
+
+- improve tests #1769
+
+- bug fixes #1766 #1772 #1773 #1775 #1776 #1777
+
+
+## 1.41.0
+
+- new apis to initiate video chats #1718 #1735
+
+- new apis `dc_msg_get_ephemeral_timer()`
+  and `dc_msg_get_ephemeral_timestamp()`
+
+- new api `dc_chatlist_get_summary2()` #1771
+
+- improve IMAP handling #1703 #1704
+
+- improve ephemeral messages #1696 #1705
+
+- mark location-messages as auto-generated #1715
+
+- multi-device avatar-sync #1716 #1717
+
+- improve python bindings #1732 #1733 #1738 #1769
+
+- Allow http scheme for DCACCOUNT urls #1770
+
+- more fixes #1702 #1706 #1707 #1710 #1719 #1721
+  #1723 #1734 #1740 #1744 #1748 #1760 #1766 #1773 #1765
+
+- refactorings #1712 #1714 #1757
+
+- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
+
+
+## 1.40.0
+
+- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
+
+- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
+
+- improve idle #1690 #1688
+
+- fix message processing issues by sequential processing #1694
+
+- refactorings #1670 #1673
+
+
+## 1.39.0
+
+- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
+
+- fix potential panics, update dependencies #1650 #1655
+
+
+## 1.38.0
+
+- fix sorting, esp. for multi-device
+
+
+## 1.37.0
+
+- improve ndn heuristics #1630
+
+- get oauth2 authorizer from provider-db #1641
+
+- removed linebreaks and spaces from generated qr-code #1631
+
+- more fixes #1633 #1635 #1636 #1637
+
+
+## 1.36.0
+
+- parse ndn (network delivery notification) reports
+  and report failed messages as such #1552 #1622 #1630
+
+- add oauth2 support for gsuite domains #1626
+
+- read image orientation from exif before recoding #1619
+
+- improve logging #1593 #1598
+
+- improve python and bot bindings #1583 #1609
+
+- improve imap logout #1595
+
+- fix sorting #1600 #1604
+
+- fix qr code generation #1631
+
+- update rustcrypto releases #1603
+
+- refactorings #1617
+
+
 ## 1.35.0
 
 - enable strict-tls from a new provider-db setting #1587

Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "deltachat"
-version = "1.35.0"
+version = "1.45.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
 [profile.release]
-# lto = true
+lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
@@ -20,10 +20,10 @@ smallvec = "1.0.0"
 surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { version = "0.3" }
+async-smtp = "0.3"
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.3.1"
+async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
 async-std = { version = "1.6.1", features = ["unstable"] }
 base64 = "0.12"
@@ -33,6 +33,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
+kamadak-exif = "0.5"
 lazy_static = "1.4.0"
 regex = "1.1.6"
 rusqlite = { version = "0.23", features = ["bundled"] }
@@ -42,8 +43,7 @@ strum = "0.18.0"
 strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
-itertools = "0.8.0"
-image-meta = "0.1.0"
+itertools = "0.9.0"
 quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
@@ -58,11 +58,16 @@ thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
+async-std-resolver = "0.19.5"
+async-tar = "0.3.0"
+uuid = { version = "0.8", features = ["serde", "v4"] }
 
 pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+dirs = { version = "3.0.1", optional=true }
+toml = "0.5.6"
 
 
 [dev-dependencies]
@@ -93,7 +98,7 @@ required-features = ["repl"]
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 

README.md

@@ -95,7 +95,8 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-
+- `RUST_LOG=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+SMTP tracing in addition to info messages.
 
 ### Expensive tests
 
@@ -123,6 +124,7 @@ Language bindings are available for:
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
 - [Go](https://github.com/hugot/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

ci_scripts/docker-coredeps/Dockerfile

@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust nightly
+# Install Rust
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share

ci_scripts/run_all.sh

@@ -15,6 +15,7 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
+export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.35.0"
+version = "1.45.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -23,6 +23,7 @@ serde_json = "1.0"
 async-std = "1.6.0"
 anyhow = "1.0.28"
 thiserror = "1.0.14"
+rand = "0.7.3"
 
 [features]
 default = ["vendored"]

deltachat-ffi/build.rs

@@ -19,10 +19,10 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
+        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();

deltachat-ffi/src/dc_array.rs

@@ -1,46 +1,56 @@
+use crate::chat::ChatItem;
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
+use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
+    MsgIds(Vec<MsgId>),
+    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub fn new(capacity: usize) -> Self {
-        dc_array_t::Uint(Vec::with_capacity(capacity))
-    }
-
-    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
-    pub fn new_locations(capacity: usize) -> Self {
-        dc_array_t::Locations(Vec::with_capacity(capacity))
-    }
-
-    pub fn add_id(&mut self, item: u32) {
-        if let Self::Uint(array) = self {
-            array.push(item);
-        } else {
-            panic!("Attempt to add id to array of other type");
-        }
-    }
-
-    pub fn add_location(&mut self, location: Location) {
-        if let Self::Locations(array) = self {
-            array.push(location)
-        } else {
-            panic!("Attempt to add a location to array of other type");
-        }
-    }
-
-    pub fn get_id(&self, index: usize) -> u32 {
+    pub(crate) fn get_id(&self, index: usize) -> u32 {
         match self {
+            Self::MsgIds(array) => array[index].to_u32(),
+            Self::Chat(array) => match array[index] {
+                ChatItem::Message { msg_id } => msg_id.to_u32(),
+                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
+                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
+            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index] as u32,
+            Self::Uint(array) => array[index],
         }
     }
 
-    pub fn get_location(&self, index: usize) -> &Location {
+    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(array) => array.get(index).and_then(|item| match item {
+                ChatItem::Message { .. } => None,
+                ChatItem::Marker1 { .. } => None,
+                ChatItem::DayMarker { timestamp } => Some(*timestamp),
+            }),
+            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(_) => None,
+            Self::Locations(array) => array
+                .get(index)
+                .and_then(|location| location.marker.as_deref()),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -48,55 +58,18 @@ impl dc_array_t {
         }
     }
 
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Locations(array) => array.is_empty(),
-            Self::Uint(array) => array.is_empty(),
-        }
-    }
-
     /// Returns the number of elements in the array.
-    pub fn len(&self) -> usize {
+    pub(crate) fn len(&self) -> usize {
         match self {
+            Self::MsgIds(array) => array.len(),
+            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub fn clear(&mut self) {
-        match self {
-            Self::Locations(array) => array.clear(),
-            Self::Uint(array) => array.clear(),
-        }
-    }
-
-    pub fn search_id(&self, needle: u32) -> Option<usize> {
-        if let Self::Uint(array) = self {
-            for (i, &u) in array.iter().enumerate() {
-                if u == needle {
-                    return Some(i);
-                }
-            }
-            None
-        } else {
-            panic!("Attempt to search for id in array of other type");
-        }
-    }
-
-    pub fn sort_ids(&mut self) {
-        if let dc_array_t::Uint(v) = self {
-            v.sort();
-        } else {
-            panic!("Attempt to sort array of something other than uints");
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const u32 {
-        if let dc_array_t::Uint(v) = self {
-            v.as_ptr()
-        } else {
-            panic!("Attempt to convert array of something other than uints to raw");
-        }
+    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
+        (0..self.len()).find(|i| self.get_id(*i) == needle)
     }
 }
 
@@ -106,6 +79,18 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
+impl From<Vec<MsgId>> for dc_array_t {
+    fn from(array: Vec<MsgId>) -> Self {
+        dc_array_t::MsgIds(array)
+    }
+}
+
+impl From<Vec<ChatItem>> for dc_array_t {
+    fn from(array: Vec<ChatItem>) -> Self {
+        dc_array_t::Chat(array)
+    }
+}
+
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -118,12 +103,11 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let mut arr = dc_array_t::new(7);
-        assert!(arr.is_empty());
+        let arr: dc_array_t = Vec::<u32>::new().into();
+        assert!(arr.len() == 0);
 
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
 
         assert_eq!(arr.len(), 1000);
 
@@ -131,31 +115,15 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        arr.clear();
-
-        assert!(arr.is_empty());
-
-        arr.add_id(13);
-        arr.add_id(7);
-        arr.add_id(666);
-        arr.add_id(0);
-        arr.add_id(5000);
-
-        arr.sort_ids();
-
-        assert_eq!(arr.get_id(0), 0);
-        assert_eq!(arr.get_id(1), 7);
-        assert_eq!(arr.get_id(2), 13);
-        assert_eq!(arr.get_id(3), 666);
+        assert_eq!(arr.search_id(10), Some(8));
+        assert_eq!(arr.search_id(1), None);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let mut arr = dc_array_t::new(7);
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/lib.rs

@@ -1,3 +1,4 @@
+#![deny(clippy::all)]
 #![allow(
     non_camel_case_types,
     non_snake_case,
@@ -23,10 +24,12 @@ use std::time::{Duration, SystemTime};
 use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
+use deltachat::accounts::Accounts;
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
+use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock::StockMessage;
@@ -36,6 +39,7 @@ mod dc_array;
 
 mod string;
 use self::string::*;
+use deltachat::chatlist::Chatlist;
 
 // as C lacks a good and portable error handling,
 // in general, the C Interface is forgiving wrt to bad parameters.
@@ -72,13 +76,17 @@ pub unsafe extern "C" fn dc_context_new(
     };
 
     let ctx = if blobdir.is_null() || *blobdir == 0 {
-        block_on(Context::new(os_name, as_path(dbfile).to_path_buf().into()))
-    } else {
-        block_on(Context::with_blobdir(
+        use rand::Rng;
+        // generate random ID as this functionality is not yet available on the C-api.
+        let id = rand::thread_rng().gen();
+        block_on(Context::new(
             os_name,
             as_path(dbfile).to_path_buf().into(),
-            as_path(blobdir).to_path_buf().into(),
+            id,
         ))
+    } else {
+        eprintln!("blobdir can not be defined explicitly anymore");
+        return ptr::null_mut();
     };
     match ctx {
         Ok(ctx) => Box::into_raw(Box::new(ctx)),
@@ -126,7 +134,7 @@ pub unsafe extern "C" fn dc_set_config(
         // When ctx.set_config() fails it already logged the error.
         // TODO: Context::set_config() should not log this
         Ok(key) => block_on(async move {
-            ctx.set_config(key, to_opt_string_lossy(value).as_ref().map(|x| x.as_str()))
+            ctx.set_config(key, to_opt_string_lossy(value).as_deref())
                 .await
                 .is_ok() as libc::c_int
         }),
@@ -285,7 +293,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     }
     let ctx = &*context;
 
-    block_on({ ctx.start_io() })
+    block_on(ctx.start_io())
 }
 
 #[no_mangle]
@@ -295,7 +303,17 @@ pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c
     }
     let ctx = &*context;
 
-    block_on({ ctx.is_io_running() }) as libc::c_int
+    block_on(ctx.is_io_running()) as libc::c_int
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
+    if context.is_null() {
+        return 0;
+    }
+    let ctx = &*context;
+
+    ctx.get_id() as libc::c_int
 }
 
 #[no_mangle]
@@ -329,37 +347,38 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         return 0;
     }
 
-    let event = &*event;
+    let event = &(*event).typ;
     match event {
-        Event::Info(_)
-        | Event::SmtpConnected(_)
-        | Event::ImapConnected(_)
-        | Event::SmtpMessageSent(_)
-        | Event::ImapMessageDeleted(_)
-        | Event::ImapMessageMoved(_)
-        | Event::ImapFolderEmptied(_)
-        | Event::NewBlobFile(_)
-        | Event::DeletedBlobFile(_)
-        | Event::Warning(_)
-        | Event::Error(_)
-        | Event::ErrorNetwork(_)
-        | Event::ErrorSelfNotInGroup(_) => 0,
-        Event::MsgsChanged { chat_id, .. }
-        | Event::IncomingMsg { chat_id, .. }
-        | Event::MsgDelivered { chat_id, .. }
-        | Event::MsgFailed { chat_id, .. }
-        | Event::MsgRead { chat_id, .. }
-        | Event::ChatModified(chat_id) => chat_id.to_u32() as libc::c_int,
-        Event::ContactsChanged(id) | Event::LocationChanged(id) => {
+        EventType::Info(_)
+        | EventType::SmtpConnected(_)
+        | EventType::ImapConnected(_)
+        | EventType::SmtpMessageSent(_)
+        | EventType::ImapMessageDeleted(_)
+        | EventType::ImapMessageMoved(_)
+        | EventType::ImapFolderEmptied(_)
+        | EventType::NewBlobFile(_)
+        | EventType::DeletedBlobFile(_)
+        | EventType::Warning(_)
+        | EventType::Error(_)
+        | EventType::ErrorNetwork(_)
+        | EventType::ErrorSelfNotInGroup(_) => 0,
+        EventType::MsgsChanged { chat_id, .. }
+        | EventType::IncomingMsg { chat_id, .. }
+        | EventType::MsgDelivered { chat_id, .. }
+        | EventType::MsgFailed { chat_id, .. }
+        | EventType::MsgRead { chat_id, .. }
+        | EventType::ChatModified(chat_id)
+        | EventType::ChatEphemeralTimerModified { chat_id, .. } => chat_id.to_u32() as libc::c_int,
+        EventType::ContactsChanged(id) | EventType::LocationChanged(id) => {
             let id = id.unwrap_or_default();
             id as libc::c_int
         }
-        Event::ConfigureProgress(progress) | Event::ImexProgress(progress) => {
+        EventType::ConfigureProgress(progress) | EventType::ImexProgress(progress) => {
             *progress as libc::c_int
         }
-        Event::ImexFileWritten(_) => 0,
-        Event::SecurejoinInviterProgress { contact_id, .. }
-        | Event::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
+        EventType::ImexFileWritten(_) => 0,
+        EventType::SecurejoinInviterProgress { contact_id, .. }
+        | EventType::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
     }
 }
 
@@ -370,35 +389,36 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         return 0;
     }
 
-    let event = &*event;
+    let event = &(*event).typ;
 
     match event {
-        Event::Info(_)
-        | Event::SmtpConnected(_)
-        | Event::ImapConnected(_)
-        | Event::SmtpMessageSent(_)
-        | Event::ImapMessageDeleted(_)
-        | Event::ImapMessageMoved(_)
-        | Event::ImapFolderEmptied(_)
-        | Event::NewBlobFile(_)
-        | Event::DeletedBlobFile(_)
-        | Event::Warning(_)
-        | Event::Error(_)
-        | Event::ErrorNetwork(_)
-        | Event::ErrorSelfNotInGroup(_)
-        | Event::ContactsChanged(_)
-        | Event::LocationChanged(_)
-        | Event::ConfigureProgress(_)
-        | Event::ImexProgress(_)
-        | Event::ImexFileWritten(_)
-        | Event::ChatModified(_) => 0,
-        Event::MsgsChanged { msg_id, .. }
-        | Event::IncomingMsg { msg_id, .. }
-        | Event::MsgDelivered { msg_id, .. }
-        | Event::MsgFailed { msg_id, .. }
-        | Event::MsgRead { msg_id, .. } => msg_id.to_u32() as libc::c_int,
-        Event::SecurejoinInviterProgress { progress, .. }
-        | Event::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
+        EventType::Info(_)
+        | EventType::SmtpConnected(_)
+        | EventType::ImapConnected(_)
+        | EventType::SmtpMessageSent(_)
+        | EventType::ImapMessageDeleted(_)
+        | EventType::ImapMessageMoved(_)
+        | EventType::ImapFolderEmptied(_)
+        | EventType::NewBlobFile(_)
+        | EventType::DeletedBlobFile(_)
+        | EventType::Warning(_)
+        | EventType::Error(_)
+        | EventType::ErrorNetwork(_)
+        | EventType::ErrorSelfNotInGroup(_)
+        | EventType::ContactsChanged(_)
+        | EventType::LocationChanged(_)
+        | EventType::ConfigureProgress(_)
+        | EventType::ImexProgress(_)
+        | EventType::ImexFileWritten(_)
+        | EventType::ChatModified(_) => 0,
+        EventType::MsgsChanged { msg_id, .. }
+        | EventType::IncomingMsg { msg_id, .. }
+        | EventType::MsgDelivered { msg_id, .. }
+        | EventType::MsgFailed { msg_id, .. }
+        | EventType::MsgRead { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::SecurejoinInviterProgress { progress, .. }
+        | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
+        EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
     }
 }
 
@@ -409,44 +429,55 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         return ptr::null_mut();
     }
 
-    let event = &*event;
+    let event = &(*event).typ;
 
     match event {
-        Event::Info(msg)
-        | Event::SmtpConnected(msg)
-        | Event::ImapConnected(msg)
-        | Event::SmtpMessageSent(msg)
-        | Event::ImapMessageDeleted(msg)
-        | Event::ImapMessageMoved(msg)
-        | Event::ImapFolderEmptied(msg)
-        | Event::NewBlobFile(msg)
-        | Event::DeletedBlobFile(msg)
-        | Event::Warning(msg)
-        | Event::Error(msg)
-        | Event::ErrorNetwork(msg)
-        | Event::ErrorSelfNotInGroup(msg) => {
+        EventType::Info(msg)
+        | EventType::SmtpConnected(msg)
+        | EventType::ImapConnected(msg)
+        | EventType::SmtpMessageSent(msg)
+        | EventType::ImapMessageDeleted(msg)
+        | EventType::ImapMessageMoved(msg)
+        | EventType::ImapFolderEmptied(msg)
+        | EventType::NewBlobFile(msg)
+        | EventType::DeletedBlobFile(msg)
+        | EventType::Warning(msg)
+        | EventType::Error(msg)
+        | EventType::ErrorNetwork(msg)
+        | EventType::ErrorSelfNotInGroup(msg) => {
             let data2 = msg.to_c_string().unwrap_or_default();
             data2.into_raw()
         }
-        Event::MsgsChanged { .. }
-        | Event::IncomingMsg { .. }
-        | Event::MsgDelivered { .. }
-        | Event::MsgFailed { .. }
-        | Event::MsgRead { .. }
-        | Event::ChatModified(_)
-        | Event::ContactsChanged(_)
-        | Event::LocationChanged(_)
-        | Event::ConfigureProgress(_)
-        | Event::ImexProgress(_)
-        | Event::SecurejoinInviterProgress { .. }
-        | Event::SecurejoinJoinerProgress { .. } => ptr::null_mut(),
-        Event::ImexFileWritten(file) => {
+        EventType::MsgsChanged { .. }
+        | EventType::IncomingMsg { .. }
+        | EventType::MsgDelivered { .. }
+        | EventType::MsgFailed { .. }
+        | EventType::MsgRead { .. }
+        | EventType::ChatModified(_)
+        | EventType::ContactsChanged(_)
+        | EventType::LocationChanged(_)
+        | EventType::ConfigureProgress(_)
+        | EventType::ImexProgress(_)
+        | EventType::SecurejoinInviterProgress { .. }
+        | EventType::SecurejoinJoinerProgress { .. }
+        | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
+        EventType::ImexFileWritten(file) => {
             let data2 = file.to_c_string().unwrap_or_default();
             data2.into_raw()
         }
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_event_get_account_id(event: *mut dc_event_t) -> u32 {
+    if event.is_null() {
+        eprintln!("ignoring careless call to dc_event_get_account_id()");
+        return 0;
+    }
+
+    (*event).id
+}
+
 #[no_mangle]
 pub type dc_event_emitter_t = EventEmitter;
 
@@ -482,7 +513,7 @@ pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *
     events
         .recv_sync()
         .map(|ev| Box::into_raw(Box::new(ev)))
-        .unwrap_or_else(|| ptr::null_mut())
+        .unwrap_or_else(ptr::null_mut)
 }
 
 #[no_mangle]
@@ -554,14 +585,7 @@ pub unsafe extern "C" fn dc_get_chatlist(
     let qi = if query_id == 0 { None } else { Some(query_id) };
 
     block_on(async move {
-        match chatlist::Chatlist::try_load(
-            &ctx,
-            flags as usize,
-            qs.as_ref().map(|x| x.as_str()),
-            qi,
-        )
-        .await
-        {
+        match chatlist::Chatlist::try_load(&ctx, flags as usize, qs.as_deref(), qi).await {
             Ok(list) => {
                 let ffi_list = ChatlistWrapper { context, list };
                 Box::into_raw(Box::new(ffi_list))
@@ -712,6 +736,25 @@ pub unsafe extern "C" fn dc_send_text_msg(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_send_videochat_invitation(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_send_videochat_invitation()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        chat::send_videochat_invitation(&ctx, ChatId::new(chat_id))
+            .await
+            .map(|msg_id| msg_id.to_u32())
+            .unwrap_or_log_default(&ctx, "Failed to send video chat invitation")
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -752,13 +795,9 @@ pub unsafe extern "C" fn dc_add_device_msg(
     };
 
     block_on(async move {
-        chat::add_device_msg(
-            &ctx,
-            to_opt_string_lossy(label).as_ref().map(|x| x.as_str()),
-            msg,
-        )
-        .await
-        .unwrap_or_log_default(&ctx, "Failed to add device message")
+        chat::add_device_msg(&ctx, to_opt_string_lossy(label).as_deref(), msg)
+            .await
+            .unwrap_or_log_default(&ctx, "Failed to add device message")
     })
     .to_u32()
 }
@@ -841,14 +880,11 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
     };
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_msgs(&ctx, ChatId::new(chat_id), flags, marker_flag)
                 .await
-                .iter()
-                .map(|msg_id| msg_id.to_u32())
-                .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+                .into(),
+        ))
     })
 }
 
@@ -977,7 +1013,7 @@ pub unsafe extern "C" fn dc_get_chat_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_media(
                 &ctx,
                 ChatId::new(chat_id),
@@ -986,11 +1022,8 @@ pub unsafe extern "C" fn dc_get_chat_media(
                 or_msg_type3,
             )
             .await
-            .iter()
-            .map(|msg_id| msg_id.to_u32())
-            .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+            .into(),
+        ))
     })
 }
 
@@ -1278,7 +1311,9 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     let muteDuration = match duration {
         0 => MuteDuration::NotMuted,
         -1 => MuteDuration::Forever,
-        n if n > 0 => MuteDuration::Until(SystemTime::now() + Duration::from_secs(duration as u64)),
+        n if n > 0 => SystemTime::now()
+            .checked_add(Duration::from_secs(duration as u64))
+            .map_or(MuteDuration::Forever, MuteDuration::Until),
         _ => {
             warn!(
                 ctx,
@@ -1296,6 +1331,49 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    // Timer value 0 is returned in the rare case of a database error,
+    // but it is not dangerous since it is only meant to be used as a
+    // default when changing the value. Such errors should not be
+    // ignored when ephemeral timer value is used to construct
+    // message headers.
+    block_on(async move { ChatId::new(chat_id).get_ephemeral_timer(ctx).await })
+        .log_err(ctx, "Failed to get ephemeral timer")
+        .unwrap_or_default()
+        .to_u32()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    timer: u32,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_set_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .set_ephemeral_timer(ctx, EphemeralTimer::from_u32(timer))
+            .await
+            .log_err(ctx, "Failed to set ephemeral timer")
+            .is_ok() as libc::c_int
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_msg_info(
     context: *mut dc_context_t,
@@ -1829,7 +1907,11 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
     }
     let ctx = &*context;
 
-    block_on({ location::send_locations_to_chat(&ctx, ChatId::new(chat_id), seconds as i64) });
+    block_on(location::send_locations_to_chat(
+        &ctx,
+        ChatId::new(chat_id),
+        seconds as i64,
+    ));
 }
 
 #[no_mangle]
@@ -1987,7 +2069,7 @@ pub unsafe extern "C" fn dc_array_get_timestamp(
         return 0;
     }
 
-    (*array).get_location(index).timestamp
+    (*array).get_timestamp(index).unwrap_or_default()
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_chat_id(
@@ -2034,7 +2116,7 @@ pub unsafe extern "C" fn dc_array_get_marker(
         return std::ptr::null_mut(); // NULL explicitly defined as "no markers"
     }
 
-    if let Some(s) = &(*array).get_location(index).marker {
+    if let Some(s) = (*array).get_marker(index) {
         s.strdup()
     } else {
         std::ptr::null_mut()
@@ -2062,16 +2144,6 @@ pub unsafe extern "C" fn dc_array_search_id(
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_array_get_raw(array: *const dc_array_t) -> *const u32 {
-    if array.is_null() {
-        eprintln!("ignoring careless call to dc_array_get_raw()");
-        return ptr::null_mut();
-    }
-
-    (*array).as_ptr()
-}
-
 // Return the independent-state of the location at the given index.
 // Independent locations do not belong to the track of the user.
 // Returns 1 if location belongs to the track of the user,
@@ -2183,6 +2255,24 @@ pub unsafe extern "C" fn dc_chatlist_get_summary(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chatlist_get_summary2(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    msg_id: u32,
+) -> *mut dc_lot_t {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_chatlist_get_summary2()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+    block_on(async move {
+        let lot =
+            Chatlist::get_summary2(&ctx, ChatId::new(chat_id), MsgId::new(msg_id), None).await;
+        Box::into_raw(Box::new(lot))
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chatlist_get_context(
     chatlist: *mut dc_chatlist_t,
@@ -2380,7 +2470,7 @@ pub unsafe extern "C" fn dc_chat_get_remaining_mute_duration(chat: *mut dc_chat_
         MuteDuration::NotMuted => 0,
         MuteDuration::Forever => -1,
         MuteDuration::Until(when) => when
-            .duration_since(SystemTime::UNIX_EPOCH)
+            .duration_since(SystemTime::now())
             .map(|d| d.as_secs() as i64)
             .unwrap_or(0),
     }
@@ -2651,6 +2741,26 @@ pub unsafe extern "C" fn dc_msg_get_showpadlock(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.get_showpadlock() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_ephemeral_timer(msg: *mut dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_ephemeral_timer()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_ephemeral_timer()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_ephemeral_timestamp(msg: *mut dc_msg_t) -> i64 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_ephemeral_timer()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_ephemeral_timestamp()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_summary(
     msg: *mut dc_msg_t,
@@ -2775,6 +2885,31 @@ pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.is_setupmessage().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_videochat_url()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+
+    ffi_msg
+        .message
+        .get_videochat_url()
+        .unwrap_or_default()
+        .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_videochat_type(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_videochat_type()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_videochat_type().unwrap_or_default() as i32
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_setupcodebegin(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2812,7 +2947,7 @@ pub unsafe extern "C" fn dc_msg_set_file(
     let ffi_msg = &mut *msg;
     ffi_msg.message.set_file(
         to_string_lossy(file),
-        to_opt_string_lossy(filemime).as_ref().map(|x| x.as_str()),
+        to_opt_string_lossy(filemime).as_deref(),
     )
 }
 
@@ -3225,3 +3360,250 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
     // currently, there is nothing to free, the provider info is a static object.
     // this may change once we start localizing string.
 }
+
+// -- Accounts
+
+/// Struct representing a list of deltachat accounts.
+pub type dc_accounts_t = Accounts;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_new(
+    os_name: *const libc::c_char,
+    dbfile: *const libc::c_char,
+) -> *mut dc_accounts_t {
+    setup_panic!();
+
+    if dbfile.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_new()");
+        return ptr::null_mut();
+    }
+
+    let os_name = if os_name.is_null() {
+        String::from("DcFFI")
+    } else {
+        to_string_lossy(os_name)
+    };
+
+    let accs = block_on(Accounts::new(os_name, as_path(dbfile).to_path_buf().into()));
+
+    match accs {
+        Ok(accs) => Box::into_raw(Box::new(accs)),
+        Err(err) => {
+            eprintln!("failed to create accounts: {}", err);
+            ptr::null_mut()
+        }
+    }
+}
+
+/// Release the accounts structure.
+///
+/// This function releases the memory of the `dc_accounts_t` structure.
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_unref(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_unref()");
+        return;
+    }
+    let _ = Box::from_raw(accounts);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_account(
+    accounts: *mut dc_accounts_t,
+    id: u32,
+) -> *mut dc_context_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_account()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.get_account(id))
+        .map(|ctx| Box::into_raw(Box::new(ctx)))
+        .unwrap_or_else(std::ptr::null_mut)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_selected_account(
+    accounts: *mut dc_accounts_t,
+) -> *mut dc_context_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_selected_account()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    let ctx = block_on(accounts.get_selected_account());
+    Box::into_raw(Box::new(ctx))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_select_account(
+    accounts: *mut dc_accounts_t,
+    id: u32,
+) -> libc::c_int {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_select_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.select_account(id))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -> u32 {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_add_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+
+    block_on(accounts.add_account()).unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_remove_account(
+    accounts: *mut dc_accounts_t,
+    id: u32,
+) -> libc::c_int {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_remove_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+
+    block_on(accounts.remove_account(id))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_migrate_account(
+    accounts: *mut dc_accounts_t,
+    dbfile: *const libc::c_char,
+) -> u32 {
+    if accounts.is_null() || dbfile.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_migrate_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    let dbfile = to_string_lossy(dbfile);
+
+    block_on(accounts.migrate_account(async_std::path::PathBuf::from(dbfile)))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_all(accounts: *mut dc_accounts_t) -> *mut dc_array_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_all()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    let list = block_on(accounts.get_all());
+    let array: dc_array_t = list.into();
+
+    Box::into_raw(Box::new(array))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_import_account(
+    accounts: *mut dc_accounts_t,
+    file: *const libc::c_char,
+) -> u32 {
+    if accounts.is_null() || file.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_import_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    let file = to_string_lossy(file);
+    block_on(accounts.import_account(async_std::path::PathBuf::from(file)))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_start_io()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.start_io());
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_stop_io(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_stop_io()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.stop_io());
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_maybe_network(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_mabye_network()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.maybe_network());
+}
+
+#[no_mangle]
+pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_event_emitter(
+    accounts: *mut dc_accounts_t,
+) -> *mut dc_accounts_event_emitter_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_event_emitter()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    let emitter = block_on(accounts.get_event_emitter());
+
+    Box::into_raw(Box::new(emitter))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_event_emitter_unref(
+    emitter: *mut dc_accounts_event_emitter_t,
+) {
+    if emitter.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_event_emitter_unref()");
+        return;
+    }
+    let _ = Box::from_raw(emitter);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_next_event(
+    emitter: *mut dc_accounts_event_emitter_t,
+) -> *mut dc_event_t {
+    if emitter.is_null() {
+        return ptr::null_mut();
+    }
+    let emitter = &*emitter;
+
+    emitter
+        .recv_sync()
+        .map(|ev| Box::into_raw(Box::new(ev)))
+        .unwrap_or_else(ptr::null_mut)
+}

deltachat-ffi/src/string.rs

@@ -105,8 +105,9 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        CString::new(self.as_ref().as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         })
     }
 
@@ -122,8 +123,9 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         }),
         None => Err(CStringError::NotUnicode),
     }

draft/msgwork_new_imap_jobs.rst

@@ -0,0 +1,66 @@
+
+simplify/streamline mark-seen/delete/move/send-mdn job handling
+---------------------------------------------------------------
+
+Idea: Introduce a new "msgwork" sql table that looks very
+much like the jobs table but has a primary key "msgid"
+and no job id and no foreign-id anymore. This opens up
+bulk-processing by looking at the whole table and combining
+flag-setting to reduce imap-roundtrips and select-folder calls.
+
+Concretely, these IMAP jobs:
+
+    DeleteMsgOnImap
+    MarkseenMsgOnImap
+    MoveMsg
+
+Would be replaced by a few per-message columns in the new msgwork table:
+
+- needs_mark_seen: (bool) message shall be marked as seen on imap
+- needs_to_move: (bool) message should be moved to mvbox_folder
+- deletion_time: (target_time or 0) message shall be deleted at specified time
+- needs_send_mdn: (bool) MDN shall be sent
+
+The various places that currently add the (replaced) jobs
+would now add/modify the respective message record in the message-work table.
+
+Looking at a single message-work entry conceptually looks like this::
+
+    if msg.server_uid==0:
+        return RetryLater  # nothing can be done without server_uid
+
+    if msg.deletion_time > current_time:
+        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
+        clear(needs_deletion)
+        clear(mark_seen)
+
+    if needs_mark_seen:
+        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
+        clear(needs_mark_seen)
+
+    if needs_send_mdn:
+        schedule_smtp_send_mdn(msg)
+        clear(needs_send_mdn)
+
+    if any_flag_set():
+        retrylater
+    # remove msgwork entry from table
+
+
+Notes/Questions:
+
+- it's unclear how much we need per-message retry-time tracking/backoff
+
+- drafting bulk processing algo is useful before
+  going for the implementation, i.e. including select_folder calls etc.
+
+- maybe it's better to not have bools for the flags but
+
+  0 (no change)
+  1 (set the imap flag)
+  2 (clear the imap flag)
+
+  and design such that we can cover all imap flags.
+
+- It might not be neccessary to keep needs_send_mdn state in this table
+  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/cmdline.rs

@@ -1,8 +1,10 @@
+extern crate dirs;
+
 use std::str::FromStr;
 
 use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
+use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -17,7 +19,7 @@ use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::Event;
+use deltachat::EventType;
 use deltachat::{config, provider};
 
 /// Reset database tables.
@@ -86,7 +88,7 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -157,7 +159,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -183,7 +185,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -202,6 +204,15 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.get_viewtype() == Viewtype::VideochatInvitation {
+            format!(
+                "[VIDEOCHAT-INVITATION: {}, type={}]",
+                msg.get_videochat_url().unwrap_or_default(),
+                msg.get_videochat_type().unwrap_or_default()
+            )
+        } else {
+            "".to_string()
+        },
         if msg.is_forwarded() {
             "[FORWARDED]"
         } else {
@@ -215,7 +226,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
 async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id.is_daymarker() {
+        if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
             println!(
                 "--------------------------------------------------------------------------------"
             );
@@ -275,7 +286,9 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, &addr).await;
+            let peerstate = Peerstate::from_addr(context, &addr)
+                .await
+                .expect("peerstate error");
             if peerstate.is_some() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
@@ -359,6 +372,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
                  listmedia\n\
@@ -430,17 +444,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
             imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
-            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -488,6 +506,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
+            let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
                 &context,
                 listflags,
@@ -495,6 +514,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
+            let time_needed = std::time::SystemTime::now()
+                .duration_since(time_start)
+                .unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -553,6 +575,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
+            println!("{:?} to create this list", time_needed);
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
@@ -569,6 +592,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
+            let msglist: Vec<MsgId> = msglist
+                .into_iter()
+                .map(|x| match x {
+                    ChatItem::Message { msg_id } => msg_id,
+                    ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
+                    ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
+                })
+                .collect();
+
             let members = chat::get_chat_contacts(&context, sel_chat.id).await;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
@@ -794,6 +826,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "videochat" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
+        }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
@@ -1029,7 +1065,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         // "event" => {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
         //     let event = arg1.parse()?;
-        //     let event = Event::from_u32(event).ok_or(format_err!("Event::from_u32({})", event))?;
+        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
         //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",

examples/repl/main.rs

@@ -19,7 +19,7 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::Event;
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
@@ -34,35 +34,35 @@ mod cmdline;
 use self::cmdline::*;
 
 /// Event Handler
-fn receive_event(event: Event) {
+fn receive_event(event: EventType) {
     let yellow = Color::Yellow.normal();
     match event {
-        Event::Info(msg) => {
+        EventType::Info(msg) => {
             /* do not show the event as this would fill the screen */
             info!("{}", msg);
         }
-        Event::SmtpConnected(msg) => {
+        EventType::SmtpConnected(msg) => {
             info!("[SMTP_CONNECTED] {}", msg);
         }
-        Event::ImapConnected(msg) => {
+        EventType::ImapConnected(msg) => {
             info!("[IMAP_CONNECTED] {}", msg);
         }
-        Event::SmtpMessageSent(msg) => {
+        EventType::SmtpMessageSent(msg) => {
             info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
-        Event::Warning(msg) => {
+        EventType::Warning(msg) => {
             warn!("{}", msg);
         }
-        Event::Error(msg) => {
+        EventType::Error(msg) => {
             error!("{}", msg);
         }
-        Event::ErrorNetwork(msg) => {
+        EventType::ErrorNetwork(msg) => {
             error!("[NETWORK] msg={}", msg);
         }
-        Event::ErrorSelfNotInGroup(msg) => {
+        EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
-        Event::MsgsChanged { chat_id, msg_id } => {
+        EventType::MsgsChanged { chat_id, msg_id } => {
             info!(
                 "{}",
                 yellow.paint(format!(
@@ -71,34 +71,34 @@ fn receive_event(event: Event) {
                 ))
             );
         }
-        Event::ContactsChanged(_) => {
+        EventType::ContactsChanged(_) => {
             info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
-        Event::LocationChanged(contact) => {
+        EventType::LocationChanged(contact) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        Event::ConfigureProgress(progress) => {
+        EventType::ConfigureProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
             );
         }
-        Event::ImexProgress(progress) => {
+        EventType::ImexProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        Event::ImexFileWritten(file) => {
+        EventType::ImexFileWritten(file) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
-        Event::ChatModified(chat) => {
+        EventType::ChatModified(chat) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
@@ -158,7 +158,7 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 27] = [
     "listchats",
     "listarchived",
     "chat",
@@ -178,6 +178,7 @@ const CHAT_COMMANDS: [&str; 26] = [
     "send",
     "sendimage",
     "sendfile",
+    "videochat",
     "draft",
     "listmedia",
     "archive",
@@ -271,12 +272,12 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            receive_event(event);
+            receive_event(event.typ);
         }
     });
 

examples/simple.rs

@@ -6,20 +6,20 @@ use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::Event;
+use deltachat::EventType;
 
-fn cb(event: Event) {
+fn cb(event: EventType) {
     match event {
-        Event::ConfigureProgress(progress) => {
+        EventType::ConfigureProgress(progress) => {
             log::info!("progress: {}", progress);
         }
-        Event::Info(msg) => {
+        EventType::Info(msg) => {
             log::info!("{}", msg);
         }
-        Event::Warning(msg) => {
+        EventType::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        Event::Error(msg) | Event::ErrorNetwork(msg) => {
+        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into())
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
@@ -45,7 +45,7 @@ async fn main() {
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            cb(event);
+            cb(event.typ);
         }
     });
 

python/CHANGELOG

@@ -1,6 +1,22 @@
-0.900.0 (DRAFT)
+1.44.0
+------
+
+- fix Chat.get_mute_duration()
+
+1.40.1
 ---------------
 
+- emit "ac_member_removed" event (with 'actor' being the removed contact) 
+  for when a user leaves a group.
+
+- fix create_contact(addr) when addr is the self-contact. 
+
+
+1.40.0
+---------------
+
+- uses latest 1.40+ Delta Chat core
+
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -10,6 +26,7 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
+
 0.800.0
 -------
 

python/README.rst

@@ -7,76 +7,14 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
-Installing bindings from source (Updated: 20-Jan-2020)
-=========================================================
-
-Install Rust and Cargo first.  Deltachat needs a specific nightly
-version, the easiest is probably to first install Rust stable from
-rustup and then use this to install the correct nightly version.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then GIT clone the deltachat-core-rust repo and get the actual
-rust- and cargo-toolchain needed by deltachat::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-   rustup show
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   ./install_python_bindings.py
-
-The installation might take a while, depending on your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_NEW_TMP_EMAIL`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs. This is implemented on the server side via
-  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
+without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
+yourself <sourceinstall>`_.
 
-We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -103,6 +41,78 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
+Running tests
+=============
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+This will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real e-mail servers.
+
+.. _livetests:
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
+
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
+
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
+One hour is enough to invoke pytest and run all offline and online tests:
+
+    pytest
+
+    # or if you have installed pytest-xdist for parallel test execution
+    pytest -n6
+
+Each test run creates new accounts.
+
+
+.. _sourceinstall:
+
+Installing bindings from source (Updated: July 2020)
+=========================================================
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   python install_python_bindings.py
+
+The core compilation and bindings building might take a while,
+depending on the speed of your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+
 Code examples
 =============
 

python/doc/_templates/globaltoc.html

@@ -9,8 +9,7 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
-  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
+  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/group_tracking.py

@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
+    def ac_member_added(self, chat, contact, actor, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
+    def ac_member_removed(self, chat, contact, actor, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
+        *ac_member_added {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))
+        *ac_member_removed {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))

python/install_python_bindings.py

@@ -17,8 +17,12 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+
     if target == 'release':
+        extra = " -C lto=on -C embed-bitcode=yes"
+        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
         cmd.append("--release")
+
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)

python/src/deltachat/account.py

@@ -246,7 +246,6 @@ class Account(object):
         addr = as_dc_charpointer(addr)
         name = as_dc_charpointer(name)
         contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -607,12 +606,24 @@ class Account(object):
         self.stop_io()
 
         self.log("remove dc_context references")
-        # the dc_context_unref triggers get_next_event to return ffi.NULL
-        # which in turns makes the event thread finish execution
+
+        # if _dc_context is unref'ed the event thread should quickly
+        # receive the termination signal. However, some python code might
+        # still hold a reference and so we use a secondary signal
+        # to make sure the even thread terminates if it receives any new
+        # event, indepedently from waiting for the core to send NULL to
+        # get_next_event().
+        self._event_thread.mark_shutdown()
         self._dc_context = None
 
         self.log("wait for event thread to finish")
-        self._event_thread.wait()
+        try:
+            self._event_thread.wait(timeout=2)
+        except RuntimeError as e:
+            self.log("Waiting for event thread failed: {}".format(e))
+
+        if self._event_thread.is_alive():
+            self.log("WARN: event thread did not terminate yet, ignoring.")
 
         self._shutdown_event.set()
 

python/src/deltachat/chat.py

@@ -137,7 +137,23 @@ class Chat(object):
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
+        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
+
+    def get_ephemeral_timer(self):
+        """ get ephemeral timer.
+
+        :returns: ephemeral timer value in seconds
+        """
+        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
+
+    def set_ephemeral_timer(self, timer):
+        """ set ephemeral timer.
+
+        :param: timer value in seconds
+
+        :returns: None
+        """
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
 
     def get_type(self):
         """ (deprecated) return type of this chat.

python/src/deltachat/direct_imap.py

@@ -108,6 +108,15 @@ class DirectImap:
 
     def get_all_messages(self):
         assert not self._idling
+
+        # Flush unsolicited responses. IMAPClient has problems
+        # dealing with them: https://github.com/mjs/imapclient/issues/334
+        # When this NOOP was introduced, next FETCH returned empty
+        # result instead of a single message, even though IMAP server
+        # can only return more untagged responses than required, not
+        # less.
+        self.conn.noop()
+
         return self.conn.fetch(ALL, [FLAGS])
 
     def get_unread_messages(self):
@@ -159,9 +168,13 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            requested = [b'BODY.PEEK[]', FLAGS]
             for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[HEADER]']
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
+                    continue
+
                 flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
@@ -187,9 +200,12 @@ class DirectImap:
         """ (blocking) wait for next idle message from server. """
         assert self._idling
         self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check()
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
         if terminate:
             self.idle_done()
+        self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
     def idle_wait_for_seen(self):

python/src/deltachat/events.py

@@ -86,11 +86,11 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
+    def get_info_contains(self, regex):
+        rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
+            if rex.search(ev.data2):
                 return ev
 
     def ensure_event_not_queued(self, event_name_regex):
@@ -139,6 +139,7 @@ class EventThread(threading.Thread):
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
+        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -147,12 +148,15 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def wait(self):
+    def mark_shutdown(self):
+        self._marked_for_shutdown = True
+
+    def wait(self, timeout=None):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join()
+        self.join(timeout=timeout)
 
     def run(self):
         """ get and run events until shutdown. """
@@ -164,10 +168,12 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while 1:
+        while not self._marked_for_shutdown:
             event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
+            if self._marked_for_shutdown:
+                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are not allowed to block/last long as this
+    Hooks are generally not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,10 +31,6 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -55,19 +51,37 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
+        """ Called when an outgoing message has been delivered to SMTP.
+
+        :param message: Message that was just delivered.
+        """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
+        """ Chat was created or modified regarding membership, avatar, title.
+
+        :param chat: Chat which was modified.
+        """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
+    def ac_member_added(self, chat, contact, actor, message):
+        """ Called for each contact added to an accepted chat.
+
+        :param chat: Chat where contact was added.
+        :param contact: Contact that was added.
+        :param actor: Who added the contact (None if it was our self-addr)
+        :param message: The original system message that reports the addition.
+        """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
+    def ac_member_removed(self, chat, contact, actor, message):
+        """ Called for each contact removed from a chat.
+
+        :param chat: Chat where contact was removed.
+        :param contact: Contact that was removed.
+        :param actor: Who removed the contact (None if it was our self-addr)
+        :param message: The original system message that reports the removal.
+        """
 
 
 class Global:

python/src/deltachat/message.py

@@ -1,6 +1,7 @@
 """ The Message object. """
 
 import os
+import re
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -154,6 +155,26 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @props.with_doc
+    def ephemeral_timer(self):
+        """Ephemeral timer in seconds
+
+        :returns: timer in seconds or None if there is no timer
+        """
+        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
+        if timer:
+            return timer
+
+    @props.with_doc
+    def ephemeral_timestamp(self):
+        """UTC time when the message will be deleted.
+
+        :returns: naive datetime.datetime() object or None if the timer is not started.
+        """
+        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
+        if ts:
+            return datetime.utcfromtimestamp(ts)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -336,20 +357,43 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
+        if not res:
+            return
+        action, affected, actor = res
+        affected = msg.account.get_contact_by_addr(affected)
+        if actor == "me":
+            actor = None
+        else:
+            actor = msg.account.get_contact_by_addr(actor)
+        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
+        return "ac_member_" + res[0], d
+
+
+def extract_addr(text):
+    m = re.match(r'.*\((.+@.+)\)', text)
+    if m:
+        text = m.group(1)
+    text = text.rstrip(".")
+    return text.strip()
 
 
 def parse_system_add_remove(text):
+    """ return add/remove info from parsing the given system message text.
+
+    returns a (action, affected, actor) triple """
+
     # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
+    # Member x@y added by a@b
+    # Member With space (tmp1@x.org) removed by tmp2@x.org.
+    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+    # Group left by some one (tmp1@x.org).
+    # Group left by tmp1@x.org.
     text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")
+    m = re.match(r'member (.+) (removed|added) by (.+)', text)
+    if m:
+        affected, action, actor = m.groups()
+        return action, extract_addr(affected), extract_addr(actor)
+    if text.startswith("group left by "):
+        addr = extract_addr(text[13:])
+        if addr:
+            return "removed", addr, addr

python/src/deltachat/testplugin.py

@@ -32,6 +32,10 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
+    parser.addoption(
+        "--strict-tls", action="store_true",
+        help="Never accept invalid TLS certificates for test accounts",
+    )
 
 
 def pytest_configure(config):
@@ -152,7 +156,7 @@ class SessionLiveConfigFromURL:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
+                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
@@ -231,10 +235,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
             ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac))
+                logger = FFIEventLogger(ac)
+                logger.init_time = self.init_time
+                ac.add_account_plugin(logger)
             self._accounts.append(ac)
             return ac
 
@@ -244,10 +251,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
+            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -282,16 +286,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            if pytestconfig.getoption("--strict-tls"):
+                # Enable strict certificate checks for online accounts
+                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
@@ -332,8 +335,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
@@ -349,6 +350,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 if hasattr(acc, "_configtracker"):
                     acc._configtracker.wait_finish()
                     del acc._configtracker
+                acc.set_config("bcc_self", "0")
                 if acc.is_configured() and not acc.is_started():
                     acc.start_io()
                 print("{}: {} account was successfully setup".format(

python/tests/test_account.py

@@ -11,8 +11,21 @@ from datetime import datetime, timedelta
 
 
 @pytest.mark.parametrize("msgtext,res", [
-    ("Member Me (tmp1@x.org) removed by tmp2@x.org.", ("removed", "tmp1@x.org")),
-    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org")),
+    ("Member Me (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by me",
+        ("removed", "tmp1@x.org", "me")),
+    ("Group left by some one (tmp1@x.org).",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Group left by tmp1@x.org.",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org", "tmp2@x.org")),
+    ("Member nothing bla bla", None),
+    ("Another unknown system message", None),
 ])
 def test_parse_system_add_remove(msgtext, res):
     from deltachat.message import parse_system_add_remove
@@ -116,6 +129,11 @@ class TestOfflineContact:
         assert not contact1.is_blocked()
         assert not contact1.is_verified()
 
+    def test_create_self_contact(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact(ac1.get_config("addr"))
+        assert contact1.id == 1
+
     def test_get_contacts_and_delete(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact("some1@example.org", name="some1")
@@ -258,15 +276,23 @@ class TestOfflineChat:
     def test_mute(self, ac1):
         chat = ac1.create_group_chat(name="title1")
         assert not chat.is_muted()
+        assert chat.get_mute_duration() == 0
         chat.mute()
         assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
         chat.unmute()
         assert not chat.is_muted()
         chat.mute(50)
         assert chat.is_muted()
+        assert chat.get_mute_duration() <= 50
         with pytest.raises(ValueError):
             chat.mute(-51)
 
+        # Regression test, this caused Rust panic previously
+        chat.mute(2**63 - 1)
+        assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
+
     def test_delete_and_send_fails(self, ac1, chat1):
         chat1.delete()
         ac1._evtracker.wait_next_messages_changed()
@@ -452,12 +478,12 @@ class TestOfflineChat:
 
         class InPlugin:
             @account_hookimpl
-            def ac_member_added(self, chat, contact):
-                in_list.append(("added", chat, contact))
+            def ac_member_added(self, chat, contact, actor):
+                in_list.append(("added", chat, contact, actor))
 
             @account_hookimpl
-            def ac_member_removed(self, chat, contact):
-                in_list.append(("removed", chat, contact))
+            def ac_member_removed(self, chat, contact, actor):
+                in_list.append(("removed", chat, contact, actor))
 
         ac1.add_account_plugin(InPlugin())
 
@@ -486,10 +512,11 @@ class TestOfflineChat:
 
         assert len(in_list) == 10
         chat_contacts = chat.get_contacts()
-        for in_cmd, in_chat, in_contact in in_list:
+        for in_cmd, in_chat, in_contact, in_actor in in_list:
             assert in_cmd == "added"
             assert in_chat == chat
             assert in_contact in chat_contacts
+            assert in_actor is None
             chat_contacts.remove(in_contact)
 
         assert chat_contacts[0].id == 1  # self contact
@@ -517,7 +544,7 @@ def test_basic_imap_api(acfactory, tmpdir):
 
     imap2 = ac2.direct_imap
 
-    ac2.direct_imap.idle_start()
+    imap2.idle_start()
     chat12.send_text("hello")
     ac2._evtracker.wait_next_incoming_message()
 
@@ -723,9 +750,9 @@ class TestOnlineAccount:
 
     def test_move_works_on_self_sent(self, acfactory):
         ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True)
-        ac1.set_config("bcc_self", "1")
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure_and_start_io()
+        ac1.set_config("bcc_self", "1")
 
         chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message1")
@@ -850,6 +877,7 @@ class TestOnlineAccount:
 
         lp.sec("mark messages as seen on ac2, wait for changes on ac1")
         ac2.direct_imap.idle_start()
+        ac1.direct_imap.idle_start()
         ac2.mark_seen_messages([msg2, msg4])
         ac2.direct_imap.idle_check(terminate=True)
         lp.step("1")
@@ -858,6 +886,8 @@ class TestOnlineAccount:
             assert ev.data1 > const.DC_CHAT_ID_LAST_SPECIAL
             assert ev.data2 > const.DC_MSG_ID_LAST_SPECIAL
         lp.step("2")
+        ac1.direct_imap.idle_wait_for_seen()  # Check that ac1 marks the read receipt as read
+
         assert msg1.is_out_mdn_received()
         assert msg3.is_out_mdn_received()
 
@@ -1539,6 +1569,127 @@ class TestOnlineAccount:
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac2.create_chat(ac4)
 
+    def test_immediate_autodelete(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account(mvbox=False, move=False, sentbox=False)
+
+        # "1" means delete immediately, while "0" means do not delete
+        ac2.set_config("delete_server_after", "1")
+
+        acfactory.wait_configure_and_start_io()
+
+        imap2 = ac2.direct_imap
+        imap2.idle_start()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+        ac2.create_chat(ac1)
+
+        sent_msg = chat1.send_text("hello")
+        imap2.idle_check(terminate=False)
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        imap2.idle_check(terminate=True)
+        ac2._evtracker.get_info_contains("close/expunge succeeded")
+
+        assert len(imap2.get_all_messages()) == 0
+
+        # Mark deleted message as seen and check that read receipt arrives
+        msg.mark_seen()
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
+        assert ev.data1 == chat1.id
+        assert ev.data2 == sent_msg.id
+
+    def test_ephemeral_timer(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+        chat2 = ac2.create_chat(ac1)
+
+        lp.sec("ac1: set ephemeral timer to 60")
+        chat1.set_ephemeral_timer(60)
+
+        lp.sec("ac1: check that ephemeral timer is set for chat")
+        assert chat1.get_ephemeral_timer() == 60
+        chat1_summary = chat1.get_summary()
+        assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
+
+        lp.sec("ac2: receive system message about ephemeral timer modification")
+        ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        system_message1 = ac2._evtracker.wait_next_incoming_message()
+        assert chat2.get_ephemeral_timer() == 60
+        assert system_message1.is_system_message()
+
+        # Disabled until markers are implemented
+        # assert "Ephemeral timer: 60\n" in system_message1.get_message_info()
+
+        lp.sec("ac2: send message to ac1")
+        sent_message = chat2.send_text("message")
+        assert sent_message.ephemeral_timer == 60
+        assert "Ephemeral timer: 60\n" in sent_message.get_message_info()
+
+        # Timer is started immediately for sent messages
+        assert sent_message.ephemeral_timestamp is not None
+        assert "Expires: " in sent_message.get_message_info()
+
+        lp.sec("ac1: waiting for message from ac2")
+        text_message = ac1._evtracker.wait_next_incoming_message()
+        assert text_message.text == "message"
+        assert text_message.ephemeral_timer == 60
+        assert "Ephemeral timer: 60\n" in text_message.get_message_info()
+
+        # Timer should not start until message is displayed
+        assert text_message.ephemeral_timestamp is None
+        assert "Expires: " not in text_message.get_message_info()
+        text_message.mark_seen()
+        text_message = ac1.get_message_by_id(text_message.id)
+        assert text_message.ephemeral_timestamp is not None
+        assert "Expires: " in text_message.get_message_info()
+
+        lp.sec("ac2: set ephemeral timer to 0")
+        chat2.set_ephemeral_timer(0)
+        ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+
+        lp.sec("ac1: receive system message about ephemeral timer modification")
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        system_message2 = ac1._evtracker.wait_next_incoming_message()
+        assert system_message2.ephemeral_timer is None
+        assert "Ephemeral timer: " not in system_message2.get_message_info()
+        assert chat1.get_ephemeral_timer() == 0
+
+    def test_delete_multiple_messages(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("ac1: sending seven messages")
+        texts = ["first", "second", "third", "fourth", "fifth", "sixth", "seventh"]
+        for text in texts:
+            chat12.send_text(text)
+
+        lp.sec("ac2: waiting for all messages on the other side")
+        to_delete = []
+        for text in texts:
+            msg = ac2._evtracker.wait_next_incoming_message()
+            assert msg.text in texts
+            if text != "third":
+                to_delete.append(msg)
+
+        lp.sec("ac2: deleting all messages except third")
+        assert len(to_delete) == len(texts) - 1
+        ac2.delete_messages(to_delete)
+        for msg in to_delete:
+            ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+
+        ac2._evtracker.get_info_contains("close/expunge succeeded")
+
+        lp.sec("imap2: test that only one message is left")
+        imap2 = ac2.direct_imap
+
+        assert len(imap2.get_all_messages()) == 1
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):

python/tox.ini

@@ -7,7 +7,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

rust-toolchain

@@ -1 +1 @@
-1.43.1
+1.45.0

set_core_version.py

@@ -5,18 +5,27 @@ import sys
 import re
 import pathlib
 import subprocess
+from argparse import ArgumentParser
 
 rex = re.compile(r'version = "(\S+)"')
 
-def read_toml_version(relpath):
+
+def regex_matches(relpath, regex=rex):
     p = pathlib.Path(relpath)
     assert p.exists()
     for line in open(str(p)):
-        m = rex.match(line)
+        m = regex.match(line)
         if m is not None:
-            return m.group(1)
+            return m
+
+
+def read_toml_version(relpath):
+    res = regex_matches(relpath, rex)
+    if res is not None:
+        return res.group(1)
     raise ValueError("no version found in {}".format(relpath))
 
+
 def replace_toml_version(relpath, newversion):
     p = pathlib.Path(relpath)
     assert p.exists()
@@ -25,18 +34,28 @@ def replace_toml_version(relpath, newversion):
         for line in open(str(p)):
             m = rex.match(line)
             if m is not None:
+                print("{}: set version={}".format(relpath, newversion))
                 f.write('version = "{}"\n'.format(newversion))
             else:
                 f.write(line)
     os.rename(tmp_path, str(p))
 
-if __name__ == "__main__":
 
-    if len(sys.argv) < 2:
-        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
+def main():
+    parser = ArgumentParser(prog="set_core_version")
+    parser.add_argument("newversion")
+
+    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml"]
+    try:
+        opts = parser.parse_args()
+    except SystemExit:
+        print()
+        for x in toml_list:
             print("{}: {}".format(x, read_toml_version(x)))
+        print()
         raise SystemExit("need argument: new version, example: 1.25.0")
-    newversion = sys.argv[1]
+
+    newversion = opts.newversion
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
 
@@ -55,7 +74,10 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
+    print("running cargo check")
     subprocess.call(["cargo", "check"])
+
+    print("adding changes to git index")
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
@@ -63,3 +85,8 @@ if __name__ == "__main__":
     print("")
     print("   git tag {}".format(newversion))
     print("")
+
+
+if __name__ == "__main__":
+    main()
+

src/accounts.rs

@@ -0,0 +1,533 @@
+use std::collections::HashMap;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::task::{Context as TaskContext, Poll};
+
+use async_std::fs;
+use async_std::path::PathBuf;
+use async_std::sync::{Arc, RwLock};
+use uuid::Uuid;
+
+use anyhow::{ensure, Context as _};
+use serde::{Deserialize, Serialize};
+
+use crate::context::Context;
+use crate::error::Result;
+use crate::events::Event;
+
+/// Account manager, that can handle multiple accounts in a single place.
+#[derive(Debug, Clone)]
+pub struct Accounts {
+    dir: PathBuf,
+    config: Config,
+    accounts: Arc<RwLock<HashMap<u32, Context>>>,
+}
+
+impl Accounts {
+    /// Loads or creates an accounts folder at the given `dir`.
+    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
+        if !dir.exists().await {
+            Accounts::create(os_name, &dir).await?;
+        }
+
+        Accounts::open(dir).await
+    }
+
+    /// Creates a new default structure, including a default account.
+    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
+        fs::create_dir_all(dir)
+            .await
+            .context("failed to create folder")?;
+
+        // create default account
+        let config = Config::new(os_name.clone(), dir).await?;
+        let account_config = config.new_account(dir).await?;
+
+        Context::new(os_name, account_config.dbfile().into(), account_config.id)
+            .await
+            .context("failed to create default account")?;
+
+        Ok(())
+    }
+
+    /// Opens an existing accounts structure. Will error if the folder doesn't exist,
+    /// no account exists and no config exists.
+    pub async fn open(dir: PathBuf) -> Result<Self> {
+        ensure!(dir.exists().await, "directory does not exist");
+
+        let config_file = dir.join(CONFIG_NAME);
+        ensure!(config_file.exists().await, "accounts.toml does not exist");
+
+        let config = Config::from_file(config_file).await?;
+        let accounts = config.load_accounts().await?;
+
+        Ok(Self {
+            dir,
+            config,
+            accounts: Arc::new(RwLock::new(accounts)),
+        })
+    }
+
+    /// Get an account by its `id`:
+    pub async fn get_account(&self, id: u32) -> Option<Context> {
+        self.accounts.read().await.get(&id).cloned()
+    }
+
+    /// Get the currently selected account.
+    pub async fn get_selected_account(&self) -> Context {
+        let id = self.config.get_selected_account().await;
+        self.accounts
+            .read()
+            .await
+            .get(&id)
+            .cloned()
+            .expect("inconsistent state")
+    }
+
+    /// Select the given account.
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        self.config.select_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Add a new account.
+    pub async fn add_account(&self) -> Result<u32> {
+        let os_name = self.config.os_name().await;
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        self.accounts.write().await.insert(account_config.id, ctx);
+
+        Ok(account_config.id)
+    }
+
+    /// Remove an account.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        let ctx = self.accounts.write().await.remove(&id);
+        ensure!(ctx.is_some(), "no account with this id: {}", id);
+        let ctx = ctx.unwrap();
+        ctx.stop_io().await;
+        drop(ctx);
+
+        if let Some(cfg) = self.config.get_account(id).await {
+            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                .await
+                .context("failed to remove account data")?;
+        }
+        self.config.remove_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Migrate an existing account into this structure.
+    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
+        let blobdir = Context::derive_blobdir(&dbfile);
+
+        ensure!(
+            dbfile.exists().await,
+            "no database found: {}",
+            dbfile.display()
+        );
+        ensure!(
+            blobdir.exists().await,
+            "no blobdir found: {}",
+            blobdir.display()
+        );
+
+        let old_id = self.config.get_selected_account().await;
+
+        // create new account
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let new_dbfile = account_config.dbfile().into();
+        let new_blobdir = Context::derive_blobdir(&new_dbfile);
+
+        let res = {
+            fs::create_dir_all(&account_config.dir).await?;
+            fs::rename(&dbfile, &new_dbfile).await?;
+            fs::rename(&blobdir, &new_blobdir).await?;
+            Ok(())
+        };
+
+        match res {
+            Ok(_) => {
+                let ctx = Context::with_blobdir(
+                    self.config.os_name().await,
+                    new_dbfile,
+                    new_blobdir,
+                    account_config.id,
+                )
+                .await?;
+                self.accounts.write().await.insert(account_config.id, ctx);
+                Ok(account_config.id)
+            }
+            Err(err) => {
+                // remove temp account
+                fs::remove_dir_all(async_std::path::PathBuf::from(&account_config.dir))
+                    .await
+                    .context("failed to remove account data")?;
+
+                self.config.remove_account(account_config.id).await?;
+
+                // set selection back
+                self.select_account(old_id).await?;
+
+                Err(err)
+            }
+        }
+    }
+
+    /// Get a list of all account ids.
+    pub async fn get_all(&self) -> Vec<u32> {
+        self.accounts.read().await.keys().copied().collect()
+    }
+
+    /// Import a backup using a new account and selects it.
+    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
+        let old_id = self.config.get_selected_account().await;
+
+        let id = self.add_account().await?;
+        let ctx = self.get_account(id).await.expect("just added");
+
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, Some(file)).await {
+            Ok(_) => Ok(id),
+            Err(err) => {
+                // remove temp account
+                self.remove_account(id).await?;
+                // set selection back
+                self.select_account(old_id).await?;
+                Err(err)
+            }
+        }
+    }
+
+    pub async fn start_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.start_io().await;
+        }
+    }
+
+    pub async fn stop_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.stop_io().await;
+        }
+    }
+
+    pub async fn maybe_network(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.maybe_network().await;
+        }
+    }
+
+    /// Unified event emitter.
+    pub async fn get_event_emitter(&self) -> EventEmitter {
+        let emitters = self
+            .accounts
+            .read()
+            .await
+            .iter()
+            .map(|(id, a)| EmitterWrapper {
+                id: *id,
+                emitter: a.get_event_emitter(),
+                done: AtomicBool::new(false),
+            })
+            .collect();
+
+        EventEmitter(emitters)
+    }
+}
+
+impl EventEmitter {
+    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
+    pub fn recv_sync(&self) -> Option<Event> {
+        async_std::task::block_on(self.recv())
+    }
+
+    /// Async recv of an event. Return `None` if the `Sender` has been droped.
+    pub async fn recv(&self) -> Option<Event> {
+        futures::future::poll_fn(|cx| Pin::new(self).recv_poll(cx)).await
+    }
+
+    fn recv_poll(self: Pin<&Self>, _cx: &mut TaskContext<'_>) -> Poll<Option<Event>> {
+        for e in &*self.0 {
+            if e.done.load(Ordering::Acquire) {
+                // skip emitters that are already done
+                continue;
+            }
+
+            match e.emitter.try_recv() {
+                Ok(event) => return Poll::Ready(Some(event)),
+                Err(async_std::sync::TryRecvError::Disconnected) => {
+                    e.done.store(true, Ordering::Release);
+                }
+                Err(async_std::sync::TryRecvError::Empty) => {}
+            }
+        }
+
+        Poll::Pending
+    }
+}
+
+#[derive(Debug)]
+pub struct EventEmitter(Vec<EmitterWrapper>);
+
+#[derive(Debug)]
+struct EmitterWrapper {
+    id: u32,
+    emitter: crate::events::EventEmitter,
+    done: AtomicBool,
+}
+
+pub const CONFIG_NAME: &str = "accounts.toml";
+pub const DB_NAME: &str = "dc.db";
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    file: PathBuf,
+    inner: Arc<RwLock<InnerConfig>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+struct InnerConfig {
+    pub os_name: String,
+    /// The currently selected account.
+    pub selected_account: u32,
+    pub next_id: u32,
+    pub accounts: Vec<AccountConfig>,
+}
+
+impl Config {
+    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
+        let cfg = Config {
+            file: dir.join(CONFIG_NAME),
+            inner: Arc::new(RwLock::new(InnerConfig {
+                os_name,
+                accounts: Vec::new(),
+                selected_account: 0,
+                next_id: 1,
+            })),
+        };
+
+        cfg.sync().await?;
+
+        Ok(cfg)
+    }
+
+    pub async fn os_name(&self) -> String {
+        self.inner.read().await.os_name.clone()
+    }
+
+    /// Sync the inmemory representation to disk.
+    async fn sync(&self) -> Result<()> {
+        fs::write(
+            &self.file,
+            toml::to_string_pretty(&*self.inner.read().await)?,
+        )
+        .await
+        .context("failed to write config")
+    }
+
+    /// Read a configuration from the given file into memory.
+    pub async fn from_file(file: PathBuf) -> Result<Self> {
+        let bytes = fs::read(&file).await.context("failed to read file")?;
+        let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
+
+        Ok(Config {
+            file,
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+
+    pub async fn load_accounts(&self) -> Result<HashMap<u32, Context>> {
+        let cfg = &*self.inner.read().await;
+        let mut accounts = HashMap::with_capacity(cfg.accounts.len());
+        for account_config in &cfg.accounts {
+            let ctx = Context::new(
+                cfg.os_name.clone(),
+                account_config.dbfile().into(),
+                account_config.id,
+            )
+            .await?;
+            accounts.insert(account_config.id, ctx);
+        }
+
+        Ok(accounts)
+    }
+
+    /// Create a new account in the given root directory.
+    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+        let id = {
+            let inner = &mut self.inner.write().await;
+            let id = inner.next_id;
+            let uuid = Uuid::new_v4();
+            let target_dir = dir.join(uuid.to_simple_ref().to_string());
+
+            inner.accounts.push(AccountConfig {
+                id,
+                name: String::new(),
+                dir: target_dir.into(),
+                uuid,
+            });
+            inner.next_id += 1;
+            id
+        };
+
+        self.sync().await?;
+
+        self.select_account(id).await.expect("just added");
+        let cfg = self.get_account(id).await.expect("just added");
+        Ok(cfg)
+    }
+
+    /// Removes an existing acccount entirely.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
+                // remove account from the configs
+                inner.accounts.remove(idx);
+            }
+            if inner.selected_account == id {
+                // reset selected account
+                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+            }
+        }
+
+        self.sync().await
+    }
+
+    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+        self.inner
+            .read()
+            .await
+            .accounts
+            .iter()
+            .find(|e| e.id == id)
+            .cloned()
+    }
+
+    pub async fn get_selected_account(&self) -> u32 {
+        self.inner.read().await.selected_account
+    }
+
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            ensure!(
+                inner.accounts.iter().any(|e| e.id == id),
+                "invalid account id: {}",
+                id
+            );
+
+            inner.selected_account = id;
+        }
+
+        self.sync().await?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct AccountConfig {
+    /// Unique id.
+    pub id: u32,
+    /// Display name
+    pub name: String,
+    /// Root directory for all data for this account.
+    pub dir: std::path::PathBuf,
+    pub uuid: Uuid,
+}
+
+impl AccountConfig {
+    /// Get the canoncial dbfile name for this configuration.
+    pub fn dbfile(&self) -> std::path::PathBuf {
+        self.dir.join(DB_NAME)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_account_new_open() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts1").into();
+
+        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let accounts2 = Accounts::open(p).await.unwrap();
+
+        assert_eq!(accounts1.accounts.read().await.len(), 1);
+        assert_eq!(accounts1.config.get_selected_account().await, 1);
+
+        assert_eq!(accounts1.dir, accounts2.dir);
+        assert_eq!(
+            &*accounts1.config.inner.read().await,
+            &*accounts2.config.inner.read().await,
+        );
+        assert_eq!(
+            accounts1.accounts.read().await.len(),
+            accounts2.accounts.read().await.len()
+        );
+    }
+
+    #[async_std::test]
+    async fn test_account_new_add_remove() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, 2);
+        assert_eq!(accounts.config.get_selected_account().await, id);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+
+        accounts.select_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        accounts.remove_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+    }
+
+    #[async_std::test]
+    async fn test_migrate_account() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let extern_dbfile: PathBuf = dir.path().join("other").into();
+        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
+            .await
+            .unwrap();
+        ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
+            .await
+            .unwrap();
+
+        drop(ctx);
+
+        accounts
+            .migrate_account(extern_dbfile.clone())
+            .await
+            .unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+
+        let ctx = accounts.get_selected_account().await;
+        assert_eq!(
+            "me@mail.com",
+            ctx.get_config(crate::config::Config::Addr).await.unwrap()
+        );
+    }
+}

src/blob.rs

@@ -14,7 +14,8 @@ use thiserror::Error;
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::events::Event;
+use crate::error::Error;
+use crate::events::EventType;
 use crate::message;
 
 /// Represents a file in the blob directory.
@@ -66,7 +67,7 @@ impl<'a> BlobObject<'a> {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -154,7 +155,7 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -167,6 +168,9 @@ impl<'a> BlobObject<'a> {
     /// subdirectory is used and [BlobObject::sanitise_name] does not
     /// modify the filename.
     ///
+    /// Paths into the blob directory may be either defined by an absolute path
+    /// or by the relative prefix `$BLOBDIR`.
+    ///
     /// # Errors
     ///
     /// This merely delegates to the [BlobObject::create_and_copy] and
@@ -178,6 +182,11 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'_>, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
+        } else if src.as_ref().starts_with("$BLOBDIR/") {
+            BlobObject::from_name(
+                context,
+                src.as_ref().to_str().unwrap_or_default().to_string(),
+            )
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -408,7 +417,13 @@ impl<'a> BlobObject<'a> {
             return Ok(());
         }
 
-        let img = img.thumbnail(img_wh, img_wh);
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
 
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
@@ -418,6 +433,24 @@ impl<'a> BlobObject<'a> {
 
         Ok(())
     }
+
+    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exifreader = exif::Reader::new();
+        let exif = exifreader.read_from_container(&mut bufreader)?;
+        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
+            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
+            // we only use rotation, in practise, flipping is not used.
+            match orientation.value.get_uint(0) {
+                Some(3) => return Ok(180),
+                Some(6) => return Ok(90),
+                Some(8) => return Ok(270),
+                other => warn!(context, "exif orientation value ignored: {:?}", other),
+            }
+        }
+        Ok(0)
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -472,7 +505,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
         let fname = t.ctx.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
@@ -483,7 +516,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_lowercase_ext() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
             .await
             .unwrap();
@@ -492,7 +525,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_file_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -501,7 +534,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_rel_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -510,7 +543,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_suffix() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -521,7 +554,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_dup() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -545,7 +578,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_double_ext_preserved() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
             .await
             .unwrap();
@@ -570,7 +603,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_long_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let s = "1".repeat(150);
         let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
@@ -579,7 +612,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_and_copy() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
         let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
@@ -595,7 +628,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_from_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
@@ -613,7 +646,7 @@ mod tests {
     }
     #[async_std::test]
     async fn test_create_from_name_long() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
         let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
@@ -635,8 +668,40 @@ mod tests {
 
     #[test]
     fn test_sanitise_name() {
-        let (_, ext) =
+        let (stem, ext) =
             BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
         assert_eq!(ext, ".txt");
+        assert!(!stem.is_empty());
+
+        // the extensions are kept together as between stem and extension a number may be added -
+        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
+        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
+        assert_eq!(stem, "wot");
+        assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
+        assert_eq!(stem, "");
+        assert_eq!(ext, ".foo.bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
+        assert!(stem.contains("foo"));
+        assert!(!stem.contains("?"));
+        assert_eq!(ext, ".bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("no-extension");
+        assert_eq!(stem, "no-extension");
+        assert_eq!(ext, "");
+
+        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
+        assert_eq!(ext, ".c");
+        assert!(!stem.contains("path"));
+        assert!(!stem.contains("ignored"));
+        assert!(stem.contains("this"));
+        assert!(stem.contains("forbidden"));
+        assert!(!stem.contains("/"));
+        assert!(!stem.contains("\\"));
+        assert!(!stem.contains(":"));
+        assert!(!stem.contains("*"));
+        assert!(!stem.contains("?"));
     }
 }

src/chat.rs

@@ -15,8 +15,9 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::{delete_expired_messages, schedule_ephemeral_task, Timer as EphemeralTimer};
 use crate::error::{bail, ensure, format_err, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Action};
 use crate::message::{self, InvalidMsgId, Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -24,6 +25,25 @@ use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
+/// An chat item, such as a message or a marker.
+#[derive(Debug, Copy, Clone)]
+pub enum ChatItem {
+    Message {
+        msg_id: MsgId,
+    },
+
+    /// A marker without inherent meaning. It is inserted before user
+    /// supplied MsgId.
+    Marker1,
+
+    /// Day marker, separating messages that correspond to different
+    /// days according to local time.
+    DayMarker {
+        /// Marker timestamp, for day markers
+        timestamp: i64,
+    },
+}
+
 /// Chat ID, including reserved IDs.
 ///
 /// Some chat IDs are reserved to identify special chat types.  This
@@ -165,7 +185,7 @@ impl ChatId {
             )
             .await?;
 
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             msg_id: MsgId::new(0),
             chat_id: ChatId::new(0),
         });
@@ -222,7 +242,7 @@ impl ChatId {
             .execute("DELETE FROM chats WHERE id=?;", paramsv![self])
             .await?;
 
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             msg_id: MsgId::new(0),
             chat_id: ChatId::new(0),
         });
@@ -248,7 +268,7 @@ impl ChatId {
         };
 
         if changed {
-            context.emit_event(Event::MsgsChanged {
+            context.emit_event(EventType::MsgsChanged {
                 chat_id: self,
                 msg_id: MsgId::new(0),
             });
@@ -424,7 +444,7 @@ impl ChatId {
     async fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
         let collect =
             |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?));
-        let (packed, rfc724_mid, mime_in_reply_to, mime_references): (
+        let (rfc724_mid, mime_in_reply_to, mime_references, error): (
             String,
             String,
             String,
@@ -432,15 +452,14 @@ impl ChatId {
         ) = self
             .parent_query(
                 context,
-                "param, rfc724_mid, mime_in_reply_to, mime_references",
+                "rfc724_mid, mime_in_reply_to, mime_references, error",
                 collect,
             )
             .await
             .ok()
             .flatten()?;
 
-        let param = packed.parse::<Params>().ok()?;
-        if param.exists(Param::Error) {
+        if !error.is_empty() {
             // Do not reply to error messages.
             //
             // An error message could be a group chat message that we failed to decrypt and
@@ -454,12 +473,13 @@ impl ChatId {
     }
 
     async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
-        let collect = |row: &rusqlite::Row| Ok(row.get(0)?);
-        let packed: Option<String> = self.parent_query(context, "param", collect).await?;
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?));
+        let res: Option<(String, String)> =
+            self.parent_query(context, "param, error", collect).await?;
 
-        if let Some(ref packed) = packed {
+        if let Some((ref packed, ref error)) = res {
             let param = packed.parse::<Params>()?;
-            Ok(!param.exists(Param::Error) && param.exists(Param::GuaranteeE2ee))
+            Ok(error.is_empty() && param.exists(Param::GuaranteeE2ee))
         } else {
             // No messages
             Ok(false)
@@ -710,6 +730,7 @@ impl Chat {
                 .unwrap_or_else(std::path::PathBuf::new),
             draft,
             is_muted: self.is_muted(),
+            ephemeral_timer: self.id.get_ephemeral_timer(context).await?,
         })
     }
 
@@ -768,7 +789,7 @@ impl Chat {
         {
             emit_event!(
                 context,
-                Event::ErrorSelfNotInGroup("Cannot send message; self not in group.".into())
+                EventType::ErrorSelfNotInGroup("Cannot send message; self not in group.".into())
             );
             bail!("Cannot set message; self not in group.");
         }
@@ -886,11 +907,10 @@ impl Chat {
 
                     // the whole list of messages referenced may be huge;
                     // only use the oldest and and the parent message
-                    let parent_references = if let Some(n) = parent_references.find(' ') {
-                        &parent_references[0..n]
-                    } else {
-                        &parent_references
-                    };
+                    let parent_references = parent_references
+                        .find(' ')
+                        .and_then(|n| parent_references.get(..n))
+                        .unwrap_or(&parent_references);
 
                     if !parent_references.is_empty() && !parent_rfc724_mid.is_empty() {
                         // angle brackets are added by the mimefactory later
@@ -938,10 +958,20 @@ impl Chat {
                     .await?;
             }
 
+            let ephemeral_timer = if msg.param.get_cmd() == SystemMessage::EphemeralTimerChanged {
+                EphemeralTimer::Disabled
+            } else {
+                self.id.get_ephemeral_timer(context).await?
+            };
+            let ephemeral_timestamp = match ephemeral_timer {
+                EphemeralTimer::Disabled => 0,
+                EphemeralTimer::Enabled { duration } => timestamp + i64::from(duration),
+            };
+
             // add message to the database
 
             if context.sql.execute(
-                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?);",
+                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id, ephemeral_timer, ephemeral_timestamp) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?);",
                         paramsv![
                             new_rfc724_mid,
                             self.id,
@@ -956,6 +986,8 @@ impl Chat {
                             new_in_reply_to,
                             new_references,
                             location_id as i32,
+                            ephemeral_timer,
+                            ephemeral_timestamp
                         ]
                     ).await.is_ok() {
                         msg_id = context.sql.get_rowid(
@@ -974,6 +1006,7 @@ impl Chat {
         } else {
             error!(context, "Cannot send message, not configured.",);
         }
+        schedule_ephemeral_task(context).await;
 
         Ok(MsgId::new(msg_id))
     }
@@ -1001,13 +1034,13 @@ impl rusqlite::types::ToSql for ChatVisibility {
 
 impl rusqlite::types::FromSql for ChatVisibility {
     fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-        i64::column_result(value).and_then(|val| {
+        i64::column_result(value).map(|val| {
             match val {
-                2 => Ok(ChatVisibility::Pinned),
-                1 => Ok(ChatVisibility::Archived),
-                0 => Ok(ChatVisibility::Normal),
+                2 => ChatVisibility::Pinned,
+                1 => ChatVisibility::Archived,
+                0 => ChatVisibility::Normal,
                 // fallback to to Normal for unknown values, may happen eg. on imports created by a newer version.
-                _ => Ok(ChatVisibility::Normal),
+                _ => ChatVisibility::Normal,
             }
         })
     }
@@ -1070,6 +1103,9 @@ pub struct ChatInfo {
     ///
     /// The exact time its muted can be found out via the `chat.mute_duration` property
     pub is_muted: bool,
+
+    /// Ephemeral message timer.
+    pub ephemeral_timer: EphemeralTimer,
     // ToDo:
     // - [ ] deaddrop,
     // - [ ] summary,
@@ -1110,7 +1146,7 @@ pub async fn create_by_msg_id(context: &Context, msg_id: MsgId) -> Result<ChatId
         chat.id.unblock(context).await;
 
         // Sending with 0s as data since multiple messages may have changed.
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -1152,7 +1188,7 @@ pub async fn create_by_contact_id(context: &Context, contact_id: u32) -> Result<
         }
     };
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -1319,7 +1355,7 @@ pub async fn prepare_msg(
 
     msg.state = MessageState::OutPreparing;
     let msg_id = prepare_msg_common(context, chat_id, msg).await?;
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: msg.chat_id,
         msg_id: msg.id,
     });
@@ -1338,11 +1374,12 @@ pub(crate) fn msgtype_has_file(msgtype: Viewtype) -> bool {
         Viewtype::Voice => true,
         Viewtype::Video => true,
         Viewtype::File => true,
+        Viewtype::VideochatInvitation => false,
     }
 }
 
 async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Error> {
-    if msg.viewtype == Viewtype::Text {
+    if msg.viewtype == Viewtype::Text || msg.viewtype == Viewtype::VideochatInvitation {
         // the caller should check if the message text is empty
     } else if msgtype_has_file(msg.viewtype) {
         let blob = msg
@@ -1464,7 +1501,7 @@ pub async fn send_msg(
                 }
             }
             msg.param.remove(Param::PrepForwards);
-            msg.save_param_to_disk(context).await;
+            msg.update_param(context).await;
         }
         return send_msg_inner(context, chat_id, msg).await;
     }
@@ -1493,7 +1530,7 @@ pub async fn send_msg_sync(
 
         match status {
             job::Status::Finished(Ok(_)) => {
-                context.emit_event(Event::MsgsChanged {
+                context.emit_event(EventType::MsgsChanged {
                     chat_id: msg.chat_id,
                     msg_id: msg.id,
                 });
@@ -1521,13 +1558,13 @@ async fn send_msg_inner(
     if let Some(send_job) = prepare_send_msg(context, chat_id, msg).await? {
         job::add(context, send_job).await;
 
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: msg.chat_id,
             msg_id: msg.id,
         });
 
         if msg.param.exists(Param::SetLatitude) {
-            context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         }
     }
 
@@ -1575,17 +1612,54 @@ pub async fn send_text_msg(
     send_msg(context, chat_id, &mut msg).await
 }
 
+pub async fn send_videochat_invitation(context: &Context, chat_id: ChatId) -> Result<MsgId, Error> {
+    ensure!(
+        !chat_id.is_special(),
+        "video chat invitation cannot be sent to special chat: {}",
+        chat_id
+    );
+
+    let instance = if let Some(instance) = context.get_config(Config::WebrtcInstance).await {
+        if !instance.is_empty() {
+            instance
+        } else {
+            bail!("webrtc_instance is empty");
+        }
+    } else {
+        bail!("webrtc_instance not set");
+    };
+
+    let instance = Message::create_webrtc_instance(&instance, &dc_create_id());
+
+    let mut msg = Message::new(Viewtype::VideochatInvitation);
+    msg.param.set(Param::WebrtcRoom, &instance);
+    msg.text = Some(
+        context
+            .stock_string_repl_str(
+                StockMessage::VideochatInviteMsgBody,
+                Message::parse_webrtc_instance(&instance).1,
+            )
+            .await,
+    );
+    send_msg(context, chat_id, &mut msg).await
+}
+
 pub async fn get_chat_msgs(
     context: &Context,
     chat_id: ChatId,
     flags: u32,
     marker1before: Option<MsgId>,
-) -> Vec<MsgId> {
-    match delete_device_expired_messages(context).await {
+) -> Vec<ChatItem> {
+    match delete_expired_messages(context).await {
         Err(err) => warn!(context, "Failed to delete expired messages: {}", err),
         Ok(messages_deleted) => {
             if messages_deleted {
-                context.emit_event(Event::MsgsChanged {
+                // Trigger reload of chatlist.
+                //
+                // On desktop chatlist is always shown on the side,
+                // and it is important to update the last message shown
+                // there.
+                context.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 })
@@ -1603,18 +1677,20 @@ pub async fn get_chat_msgs(
             let (curr_id, ts) = row?;
             if let Some(marker_id) = marker1before {
                 if curr_id == marker_id {
-                    ret.push(MsgId::new(DC_MSG_ID_MARKER1));
+                    ret.push(ChatItem::Marker1);
                 }
             }
             if (flags & DC_GCM_ADDDAYMARKER) != 0 {
                 let curr_local_timestamp = ts + cnv_to_local;
                 let curr_day = curr_local_timestamp / 86400;
                 if curr_day != last_day {
-                    ret.push(MsgId::new(DC_MSG_ID_DAYMARKER));
+                    ret.push(ChatItem::DayMarker {
+                        timestamp: curr_day,
+                    });
                     last_day = curr_day;
                 }
             }
-            ret.push(curr_id);
+            ret.push(ChatItem::Message { msg_id: curr_id });
         }
         Ok(ret)
     };
@@ -1706,7 +1782,7 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<(),
         )
         .await?;
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -1738,7 +1814,7 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
         )
         .await?;
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         msg_id: MsgId::new(0),
         chat_id: ChatId::new(0),
     });
@@ -1746,52 +1822,6 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-/// Deletes messages which are expired according to "delete_device_after" setting.
-///
-/// Returns true if any message is deleted, so event can be emitted. If nothing
-/// has been deleted, returns false.
-pub async fn delete_device_expired_messages(context: &Context) -> Result<bool, Error> {
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
-        let threshold_timestamp = time() - delete_device_after;
-
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .await
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .await
-            .unwrap_or_default()
-            .0;
-
-        // Delete expired messages
-        //
-        // Only update the rows that have to be updated, to avoid emitting
-        // unnecessary "chat modified" events.
-        let rows_modified = context
-            .sql
-            .execute(
-                "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
-             WHERE timestamp < ? \
-             AND chat_id > ? \
-             AND chat_id != ? \
-             AND chat_id != ?",
-                paramsv![
-                    DC_CHAT_ID_TRASH,
-                    threshold_timestamp,
-                    DC_CHAT_ID_LAST_SPECIAL,
-                    self_chat_id,
-                    device_chat_id
-                ],
-            )
-            .await?;
-
-        Ok(rows_modified > 0)
-    } else {
-        Ok(false)
-    }
-}
-
 pub async fn get_chat_media(
     context: &Context,
     chat_id: ChatId,
@@ -1955,7 +1985,7 @@ pub async fn create_group_chat(
         chat_id.set_draft_raw(context, &mut draft_msg).await;
     }
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         msg_id: MsgId::new(0),
         chat_id: ChatId::new(0),
     });
@@ -2055,7 +2085,9 @@ pub(crate) async fn add_contact_to_chat_ex(
         /* we should respect this - whatever we send to the group, it gets discarded anyway! */
         emit_event!(
             context,
-            Event::ErrorSelfNotInGroup("Cannot add contact to group; self not in group.".into())
+            EventType::ErrorSelfNotInGroup(
+                "Cannot add contact to group; self not in group.".into()
+            )
         );
         bail!("can not add contact because our account is not part of it");
     }
@@ -2113,7 +2145,7 @@ pub(crate) async fn add_contact_to_chat_ex(
         msg.param.set_int(Param::Arg2, from_handshake.into());
         msg.id = send_msg(context, chat_id, &mut msg).await?;
     }
-    context.emit_event(Event::ChatModified(chat_id));
+    context.emit_event(EventType::ChatModified(chat_id));
     Ok(true)
 }
 
@@ -2275,7 +2307,7 @@ pub async fn set_muted(
         .await
         .is_ok()
     {
-        context.emit_event(Event::ChatModified(chat_id));
+        context.emit_event(EventType::ChatModified(chat_id));
     } else {
         bail!("Failed to set mute duration, chat might not exist -");
     }
@@ -2307,7 +2339,7 @@ pub async fn remove_contact_from_chat(
             if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
                 emit_event!(
                     context,
-                    Event::ErrorSelfNotInGroup(
+                    EventType::ErrorSelfNotInGroup(
                         "Cannot remove contact from chat; self not in group.".into()
                     )
                 );
@@ -2355,7 +2387,7 @@ pub async fn remove_contact_from_chat(
                 // removed it first, it would complicate the
                 // check/encryption logic.
                 success = remove_from_chat_contacts_table(context, chat_id, contact_id).await;
-                context.emit_event(Event::ChatModified(chat_id));
+                context.emit_event(EventType::ChatModified(chat_id));
             }
         }
     }
@@ -2415,7 +2447,7 @@ pub async fn set_chat_name(
         } else if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
             emit_event!(
                 context,
-                Event::ErrorSelfNotInGroup("Cannot set chat name; self not in group".into())
+                EventType::ErrorSelfNotInGroup("Cannot set chat name; self not in group".into())
             );
         } else {
             /* we should respect this - whatever we send to the group, it gets discarded anyway! */
@@ -2445,12 +2477,12 @@ pub async fn set_chat_name(
                         msg.param.set(Param::Arg, &chat.name);
                     }
                     msg.id = send_msg(context, chat_id, &mut msg).await?;
-                    context.emit_event(Event::MsgsChanged {
+                    context.emit_event(EventType::MsgsChanged {
                         chat_id,
                         msg_id: msg.id,
                     });
                 }
-                context.emit_event(Event::ChatModified(chat_id));
+                context.emit_event(EventType::ChatModified(chat_id));
                 success = true;
             }
         }
@@ -2483,7 +2515,9 @@ pub async fn set_chat_profile_image(
     if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
         emit_event!(
             context,
-            Event::ErrorSelfNotInGroup("Cannot set chat profile image; self not in group.".into())
+            EventType::ErrorSelfNotInGroup(
+                "Cannot set chat profile image; self not in group.".into()
+            )
         );
         bail!("Failed to set profile image");
     }
@@ -2522,13 +2556,13 @@ pub async fn set_chat_profile_image(
         msg.id = send_msg(context, chat_id, &mut msg).await?;
         emit_event!(
             context,
-            Event::MsgsChanged {
+            EventType::MsgsChanged {
                 chat_id,
                 msg_id: msg.id
             }
         );
     }
-    emit_event!(context, Event::ChatModified(chat_id));
+    emit_event!(context, EventType::ChatModified(chat_id));
     Ok(())
 }
 
@@ -2596,7 +2630,7 @@ pub async fn forward_msgs(
                         .set(Param::PrepForwards, new_msg_id.to_u32().to_string());
                 }
 
-                msg.save_param_to_disk(context).await;
+                msg.update_param(context).await;
                 msg.param = save_param;
             } else {
                 msg.state = MessageState::OutPending;
@@ -2612,7 +2646,7 @@ pub async fn forward_msgs(
         }
     }
     for (chat_id, msg_id) in created_chats.iter().zip(created_msgs.iter()) {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: *chat_id,
             msg_id: *msg_id,
         });
@@ -2672,10 +2706,12 @@ pub(crate) async fn get_chat_id_by_grpid(
 /// Adds a message to device chat.
 ///
 /// Optional `label` can be provided to ensure that message is added only once.
-pub async fn add_device_msg(
+/// If `important` is true, a notification will be sent.
+pub async fn add_device_msg_with_importance(
     context: &Context,
     label: Option<&str>,
     msg: Option<&mut Message>,
+    important: bool,
 ) -> Result<MsgId, Error> {
     ensure!(
         label.is_some() || msg.is_some(),
@@ -2735,12 +2771,24 @@ pub async fn add_device_msg(
     }
 
     if !msg_id.is_unset() {
-        context.emit_event(Event::IncomingMsg { chat_id, msg_id });
+        if important {
+            context.emit_event(EventType::IncomingMsg { chat_id, msg_id });
+        } else {
+            context.emit_event(EventType::MsgsChanged { chat_id, msg_id });
+        }
     }
 
     Ok(msg_id)
 }
 
+pub async fn add_device_msg(
+    context: &Context,
+    label: Option<&str>,
+    msg: Option<&mut Message>,
+) -> Result<MsgId, Error> {
+    add_device_msg_with_importance(context, label, msg, false).await
+}
+
 pub async fn was_device_msg_ever_added(context: &Context, label: &str) -> Result<bool, Error> {
     ensure!(!label.is_empty(), "empty label");
     if let Ok(()) = context
@@ -2783,9 +2831,16 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
 /// For example, it can be a message showing that a member was added to a group.
 pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
+    let ephemeral_timer = match chat_id.get_ephemeral_timer(context).await {
+        Err(e) => {
+            warn!(context, "Could not get timer for info msg: {}", e);
+            return;
+        }
+        Ok(ephemeral_timer) => ephemeral_timer,
+    };
 
-    if context.sql.execute(
-        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid) VALUES (?,?,?, ?,?,?, ?,?);",
+    if let Err(e) = context.sql.execute(
+        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid,ephemeral_timer) VALUES (?,?,?, ?,?,?, ?,?,?);",
         paramsv![
             chat_id,
             DC_CONTACT_ID_INFO,
@@ -2795,8 +2850,10 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
             MessageState::InNoticed,
             text.as_ref().to_string(),
             rfc724_mid,
+            ephemeral_timer
         ]
-    ).await.is_err() {
+    ).await {
+        warn!(context, "Could not add info msg: {}", e);
         return;
     }
 
@@ -2805,7 +2862,7 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
         .get_rowid(context, "msgs", "rfc724_mid", &rfc724_mid)
         .await
         .unwrap_or_default();
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id,
         msg_id: MsgId::new(row_id),
     });
@@ -2820,7 +2877,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_chat_info() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let bob = Contact::create(&t.ctx, "bob", "bob@example.com")
             .await
             .unwrap();
@@ -2843,7 +2900,8 @@ mod tests {
                 "color": 15895624,
                 "profile_image": "",
                 "draft": "",
-                "is_muted": false
+                "is_muted": false,
+                "ephemeral_timer": "Disabled"
             }
         "#;
 
@@ -2854,7 +2912,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_draft_no_draft() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2864,7 +2922,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_draft_special_chat_id() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let draft = ChatId::new(DC_CHAT_ID_LAST_SPECIAL)
             .get_draft(&t.ctx)
             .await
@@ -2876,14 +2934,14 @@ mod tests {
     async fn test_get_draft_no_chat() {
         // This is a weird case, maybe this should be an error but we
         // do not get this info from the database currently.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let draft = ChatId::new(42).get_draft(&t.ctx).await.unwrap();
         assert!(draft.is_none());
     }
 
     #[async_std::test]
     async fn test_get_draft() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2899,7 +2957,7 @@ mod tests {
     #[async_std::test]
     async fn test_add_contact_to_chat_ex_add_self() {
         // Adding self to a contact should succeed, even though it's pointless.
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -2911,7 +2969,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_self_talk() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
@@ -2932,7 +2990,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_deaddrop_chat() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat = Chat::load_from_db(&t.ctx, ChatId::new(DC_CHAT_ID_DEADDROP))
             .await
             .unwrap();
@@ -2947,7 +3005,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_unlabelled() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         // add two device-messages
         let mut msg1 = Message::new(Viewtype::Text);
@@ -2982,7 +3040,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_labelled() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         // add two device-messages with the same label (second attempt is not added)
         let mut msg1 = Message::new(Viewtype::Text);
@@ -3036,7 +3094,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_add_device_msg_label_only() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let res = add_device_msg(&t.ctx, Some(""), None).await;
         assert!(res.is_err());
         let res = add_device_msg(&t.ctx, Some("some-label"), None).await;
@@ -3056,7 +3114,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_was_device_msg_ever_added() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         add_device_msg(&t.ctx, Some("some-label"), None).await.ok();
         assert!(was_device_msg_ever_added(&t.ctx, "some-label")
             .await
@@ -3080,7 +3138,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_delete_device_chat() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("message text".to_string());
@@ -3100,7 +3158,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_device_chat_cannot_sent() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
         let (device_chat_id, _) =
             create_or_lookup_by_contact_id(&t.ctx, DC_CONTACT_ID_DEVICE, Blocked::Not)
@@ -3120,7 +3178,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_delete_and_reset_all_device_msgs() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("message text".to_string());
         let msg_id1 = add_device_msg(&t.ctx, Some("some-label"), Some(&mut msg))
@@ -3157,7 +3215,7 @@ mod tests {
     #[async_std::test]
     async fn test_archive() {
         // create two chats
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("foo".to_string());
         let msg_id = add_device_msg(&t.ctx, None, Some(&mut msg)).await.unwrap();
@@ -3271,7 +3329,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_pinned() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // create 3 chats, wait 1 second in between to get a reliable order (we order by time)
         let mut msg = Message::new(Viewtype::Text);
@@ -3330,7 +3388,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_chat_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3354,7 +3412,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_same_chat_twice() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let contact1 = Contact::create(&context.ctx, "bob", "bob@mail.de")
             .await
             .unwrap();
@@ -3373,7 +3431,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_shall_attach_selfavatar() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3397,7 +3455,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_mute_duration() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();
@@ -3465,7 +3523,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_parent_is_encrypted() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
             .await
             .unwrap();

src/chatlist.rs

@@ -5,6 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
+use crate::ephemeral::delete_expired_messages;
 use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
@@ -76,7 +77,7 @@ impl Chatlist {
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
     ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+    ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -99,7 +100,7 @@ impl Chatlist {
 
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context).await {
+        if let Err(err) = delete_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -147,11 +148,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
@@ -173,11 +175,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
@@ -206,11 +209,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND c.name LIKE ?3
@@ -236,11 +240,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND NOT c.archived=?3
@@ -324,20 +329,30 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let mut ret = Lot::new();
-
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
             None => {
+                let mut ret = Lot::new();
                 ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return ret;
+                return Lot::new();
             }
         };
 
+        Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
+    }
+
+    pub async fn get_summary2(
+        context: &Context,
+        chat_id: ChatId,
+        lastmsg_id: MsgId,
+        chat: Option<&Chat>,
+    ) -> Lot {
+        let mut ret = Lot::new();
+
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
+        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
@@ -346,7 +361,7 @@ impl Chatlist {
 
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
@@ -424,7 +439,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_try_load() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
@@ -472,7 +487,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
         create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
@@ -497,7 +512,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_search_special_chat_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
 
         let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
@@ -530,7 +545,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_summary_unwrap() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();

src/config.rs

@@ -8,10 +8,10 @@ use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::{scheduler::InterruptInfo, stock::StockMessage};
+use crate::stock::StockMessage;
 
 /// The available configuration keys.
 #[derive(
@@ -117,9 +117,21 @@ pub enum Config {
 
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
+
+    #[strum(props(default = "0"))]
+    /// Whether we send a warning if the password is wrong (set to false when we send a warning
+    /// because we do not want to send a second warning)
+    NotifyAboutWrongPw,
+
+    /// address to webrtc instance to use for videochats
+    WebrtcInstance,
 }
 
 impl Context {
+    pub async fn config_exists(&self, key: Config) -> bool {
+        self.sql.get_raw_config(self, key).await.is_some()
+    }
+
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
     pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
@@ -201,22 +213,6 @@ impl Context {
                     None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
-            Config::InboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_inbox(InterruptInfo::new(false, None)).await;
-                ret
-            }
-            Config::SentboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_sentbox(InterruptInfo::new(false, None))
-                    .await;
-                ret
-            }
-            Config::MvboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_mvbox(InterruptInfo::new(false, None)).await;
-                ret
-            }
             Config::Selfstatus => {
                 let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
@@ -230,7 +226,7 @@ impl Context {
             Config::DeleteDeviceAfter => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
-                self.emit_event(Event::MsgsChanged {
+                self.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 });
@@ -286,7 +282,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_outside_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         File::create(&avatar_src)
@@ -315,7 +311,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_in_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.ctx.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
@@ -341,7 +337,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
         File::create(&avatar_src)
@@ -365,7 +361,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_media_quality_config_option() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure/mod.rs

@@ -4,7 +4,7 @@ mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
 
-use anyhow::{bail, ensure, format_err, Context as _, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
@@ -28,7 +28,7 @@ macro_rules! progress {
             $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::ConfigureProgress($progress));
+        $context.emit_event($crate::events::EventType::ConfigureProgress($progress));
     };
 }
 
@@ -68,16 +68,21 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let was_configured_before = self.is_configured().await;
         let mut param = LoginParam::from_database(self, "").await;
         let success = configure(self, &mut param).await;
+        self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
         if let Some(provider) = provider::get_provider_info(&param.addr) {
-            if !was_configured_before {
-                if let Some(config_defaults) = &provider.config_defaults {
-                    for def in config_defaults.iter() {
+            if let Some(config_defaults) = &provider.config_defaults {
+                for def in config_defaults.iter() {
+                    if !self.config_exists(def.key).await {
                         info!(self, "apply config_defaults {}={}", def.key, def.value);
                         self.set_config(def.key, Some(def.value)).await?;
+                    } else {
+                        info!(
+                            self,
+                            "skip already set config_defaults {}={}", def.key, def.value
+                        );
                     }
                 }
             }
@@ -96,11 +101,12 @@ impl Context {
 
         match success {
             Ok(_) => {
+                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+                    .await?;
                 progress!(self, 1000);
                 Ok(())
             }
             Err(err) => {
-                error!(self, "Configure Failed: {}", err);
                 progress!(self, 0);
                 Err(err)
             }
@@ -186,74 +192,26 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     param.server_flags |= keep_flags;
 
     // Step 3: Fill missing fields with defaults
-    if param.mail_server.is_empty() {
-        param.mail_server = format!("imap.{}", param_domain,)
-    }
-    if param.mail_port == 0 {
-        param.mail_port = if 0 != param.server_flags & (0x100 | 0x400) {
-            143
-        } else {
-            993
-        }
-    }
-    if param.mail_user.is_empty() {
-        param.mail_user = param.addr.clone();
-    }
-    if param.send_server.is_empty() && !param.mail_server.is_empty() {
-        param.send_server = param.mail_server.clone();
-        if param.send_server.starts_with("imap.") {
-            param.send_server = param.send_server.replacen("imap", "smtp", 1);
-        }
-    }
-    if param.send_port == 0 {
-        param.send_port = if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
-            587
-        } else if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
-            25
-        } else {
-            465
-        }
-    }
-    if param.send_user.is_empty() && !param.mail_user.is_empty() {
+    if param.send_user.is_empty() {
         param.send_user = param.mail_user.clone();
     }
-    if param.send_pw.is_empty() && !param.mail_pw.is_empty() {
+    if param.send_pw.is_empty() {
         param.send_pw = param.mail_pw.clone()
     }
+    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+    }
     if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
         param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
         param.server_flags |= DC_LP_AUTH_NORMAL as i32
     }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= if param.send_port == 143 {
-            DC_LP_IMAP_SOCKET_STARTTLS as i32
-        } else {
-            DC_LP_IMAP_SOCKET_SSL as i32
-        }
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= if param.send_port == 587 {
-            DC_LP_SMTP_SOCKET_STARTTLS as i32
-        } else if param.send_port == 25 {
-            DC_LP_SMTP_SOCKET_PLAIN as i32
-        } else {
-            DC_LP_SMTP_SOCKET_SSL as i32
-        }
-    }
 
     // do we have a complete configuration?
     ensure!(
-        !param.mail_server.is_empty()
-            && param.mail_port != 0
-            && !param.mail_user.is_empty()
-            && !param.mail_pw.is_empty()
-            && !param.send_server.is_empty()
-            && param.send_port != 0
-            && !param.send_user.is_empty()
-            && !param.send_pw.is_empty()
-            && param.server_flags != 0,
+        !param.mail_pw.is_empty() && !param.send_pw.is_empty(),
         "Account settings incomplete."
     );
 
@@ -263,10 +221,23 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     let (_s, r) = async_std::sync::channel(1);
     let mut imap = Imap::new(r);
 
-    try_imap_connections(ctx, param, param_autoconfig.is_some(), &mut imap).await?;
-    progress!(ctx, 800);
+    if param_autoconfig.is_some() {
+        if try_imap_one_param(ctx, &param, &mut imap).await.is_err() {
+            bail!("IMAP autoconfig did not succeed");
+        }
+    } else {
+        *param = try_imap_hostnames(ctx, param.clone(), &mut imap).await?;
+    }
+    progress!(ctx, 750);
 
-    try_smtp_connections(ctx, param, param_autoconfig.is_some()).await?;
+    let mut smtp = Smtp::new();
+    if param_autoconfig.is_some() {
+        if try_smtp_one_param(ctx, &param, &mut smtp).await.is_err() {
+            bail!("SMTP autoconfig did not succeed");
+        }
+    } else {
+        *param = try_smtp_hostnames(ctx, param.clone(), &mut smtp).await?;
+    }
     progress!(ctx, 900);
 
     let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
@@ -405,7 +376,7 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
                         p.mail_server = imap.hostname.to_string();
                         p.mail_user = imap.apply_username_pattern(param.addr.clone());
                         p.mail_port = imap.port as i32;
-                        p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.imap_certificate_checks = CertificateChecks::Automatic;
                         p.server_flags |= match imap.socket {
                             provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
                             provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
@@ -414,7 +385,7 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
                         p.send_server = smtp.hostname.to_string();
                         p.send_user = smtp.apply_username_pattern(param.addr.clone());
                         p.send_port = smtp.port as i32;
-                        p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.smtp_certificate_checks = CertificateChecks::Automatic;
                         p.server_flags |= match smtp.socket {
                             provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
                             provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
@@ -437,77 +408,105 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
     None
 }
 
-async fn try_imap_connections(
+async fn try_imap_hostnames(
     context: &Context,
-    param: &mut LoginParam,
-    was_autoconfig: bool,
+    mut param: LoginParam,
     imap: &mut Imap,
-) -> Result<()> {
-    // manually_set_param is used to check whether a particular setting was set manually by the user.
-    // If yes, we do not want to change it to avoid confusing error messages
-    // (you set port 443, but the app tells you it couldn't connect on port 993).
-    let manually_set_param = LoginParam::from_database(context, "").await;
+) -> Result<LoginParam> {
+    if param.mail_server.is_empty() {
+        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
+        let param_domain = parsed.domain;
 
-    // progress 650 and 660
-    if try_imap_connection(context, param, &manually_set_param, was_autoconfig, 0, imap)
-        .await
-        .is_ok()
-    {
-        return Ok(()); // we directly return here if it was autoconfig or the connection succeeded
-    }
+        param.mail_server = param_domain.clone();
+        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
+            return Ok(param);
+        }
 
-    progress!(context, 670);
-    // try_imap_connection() changed the flags and port. Change them back:
-    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-        param.server_flags |= DC_LP_IMAP_SOCKET_SSL;
+        progress!(context, 650);
+        param.mail_server = "imap.".to_string() + &param_domain;
+        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
+            return Ok(param);
+        }
+
+        progress!(context, 700);
+        param.mail_server = "mail.".to_string() + &param_domain;
+        try_imap_ports(context, param, imap).await
+    } else {
+        progress!(context, 700);
+        try_imap_ports(context, param, imap).await
     }
-    if manually_set_param.mail_port == 0 {
-        param.mail_port = 993;
-    }
-    if let Some(at) = param.mail_user.find('@') {
-        param.mail_user = param.mail_user.split_at(at).0.to_string();
-    }
-    if let Some(at) = param.send_user.find('@') {
-        param.send_user = param.send_user.split_at(at).0.to_string();
-    }
-    // progress 680 and 690
-    try_imap_connection(context, param, &manually_set_param, was_autoconfig, 1, imap).await
 }
 
-async fn try_imap_connection(
+// Try various IMAP ports and corresponding TLS settings.
+async fn try_imap_ports(
     context: &Context,
-    param: &mut LoginParam,
-    manually_set_param: &LoginParam,
-    was_autoconfig: bool,
-    variation: usize,
+    mut param: LoginParam,
     imap: &mut Imap,
-) -> Result<()> {
-    if try_imap_one_param(context, param, imap).await.is_ok() {
-        return Ok(());
-    }
-    if was_autoconfig {
-        return Ok(());
-    }
-
-    progress!(context, 650 + variation * 30);
-
-    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS;
-
-        if try_imap_one_param(context, &param, imap).await.is_ok() {
-            return Ok(());
+) -> Result<LoginParam> {
+    // Try to infer port from socket security.
+    if param.mail_port == 0 {
+        if 0 != param.server_flags & DC_LP_IMAP_SOCKET_SSL {
+            param.mail_port = 993
+        }
+        if 0 != param.server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN) {
+            param.mail_port = 143
         }
     }
 
-    progress!(context, 660 + variation * 30);
+    if param.mail_port == 0 {
+        // Neither port nor security is set.
+        //
+        // Try common secure combinations.
 
-    if manually_set_param.mail_port == 0 {
+        // Try TLS over port 993
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
+        param.mail_port = 993;
+        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
+            return Ok(login_param);
+        }
+
+        // Try STARTTLS over port 143
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
         param.mail_port = 143;
-        try_imap_one_param(context, param, imap).await
+        try_imap_usernames(context, param, imap).await
+    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
+        // Try TLS over user-provided port.
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
+        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
+            return Ok(login_param);
+        }
+
+        // Try STARTTLS over user-provided port.
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
+        try_imap_usernames(context, param, imap).await
     } else {
-        Err(format_err!("no more possible configs"))
+        try_imap_usernames(context, param, imap).await
+    }
+}
+
+async fn try_imap_usernames(
+    context: &Context,
+    mut param: LoginParam,
+    imap: &mut Imap,
+) -> Result<LoginParam> {
+    if param.mail_user.is_empty() {
+        param.mail_user = param.addr.clone();
+        if let Err(e) = try_imap_one_param(context, &param, imap).await {
+            if let Some(at) = param.mail_user.find('@') {
+                param.mail_user = param.mail_user.split_at(at).0.to_string();
+                try_imap_one_param(context, &param, imap).await?;
+            } else {
+                return Err(e);
+            }
+        }
+        Ok(param)
+    } else {
+        try_imap_one_param(context, &param, imap).await?;
+        Ok(param)
     }
 }
 
@@ -530,48 +529,109 @@ async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Im
     bail!("Could not connect: {}", inf);
 }
 
-async fn try_smtp_connections(
+async fn try_smtp_hostnames(
     context: &Context,
-    param: &mut LoginParam,
-    was_autoconfig: bool,
-) -> Result<()> {
-    // manually_set_param is used to check whether a particular setting was set manually by the user.
-    // If yes, we do not want to change it to avoid confusing error messages
-    // (you set port 443, but the app tells you it couldn't connect on port 993).
-    let manually_set_param = LoginParam::from_database(context, "").await;
+    mut param: LoginParam,
+    smtp: &mut Smtp,
+) -> Result<LoginParam> {
+    if param.send_server.is_empty() {
+        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
+        let param_domain = parsed.domain;
 
-    let mut smtp = Smtp::new();
-    // try to connect to SMTP - if we did not got an autoconfig, the first try was SSL-465 and we do
-    // a second try with STARTTLS-587
-    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
-        return Ok(());
-    }
-    if was_autoconfig {
-        return Ok(());
-    }
-    progress!(context, 850);
+        param.send_server = param_domain.clone();
+        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
+            return Ok(param);
+        }
 
-    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        progress!(context, 800);
+        param.send_server = "smtp.".to_string() + &param_domain;
+        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
+            return Ok(param);
+        }
+
+        progress!(context, 850);
+        param.send_server = "mail.".to_string() + &param_domain;
+        try_smtp_ports(context, param, smtp).await
+    } else {
+        progress!(context, 850);
+        try_smtp_ports(context, param, smtp).await
+    }
+}
+
+// Try various SMTP ports and corresponding TLS settings.
+async fn try_smtp_ports(
+    context: &Context,
+    mut param: LoginParam,
+    smtp: &mut Smtp,
+) -> Result<LoginParam> {
+    // Try to infer port from socket security.
+    if param.send_port == 0 {
+        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
+            param.send_port = 587;
+        }
+        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
+            param.send_port = 25;
+        }
+        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_SSL as i32 {
+            param.send_port = 465;
+        }
+    }
+
+    if param.send_port == 0 {
+        // Neither port nor security is set.
+        //
+        // Try common secure combinations.
+
+        // Try TLS over port 465.
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
+        param.send_port = 465;
+        if let Ok(login_param) = try_smtp_usernames(context, param.clone(), smtp).await {
+            return Ok(login_param);
+        }
+
+        // Try STARTTLS over port 587.
         param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
         param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    }
-    if manually_set_param.send_port == 0 {
         param.send_port = 587;
-    }
+        try_smtp_usernames(context, param, smtp).await
+    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
+        // Try TLS over user-provided port.
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
+        if let Ok(param) = try_smtp_usernames(context, param.clone(), smtp).await {
+            return Ok(param);
+        }
 
-    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
-        return Ok(());
-    }
-    progress!(context, 860);
-
-    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        // Try STARTTLS over user-provided port.
         param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
         param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
+        try_smtp_usernames(context, param, smtp).await
+    } else {
+        try_smtp_usernames(context, param, smtp).await
     }
-    if manually_set_param.send_port == 0 {
-        param.send_port = 25;
+}
+
+async fn try_smtp_usernames(
+    context: &Context,
+    mut param: LoginParam,
+    smtp: &mut Smtp,
+) -> Result<LoginParam> {
+    if param.send_user.is_empty() {
+        param.send_user = param.addr.clone();
+        if let Err(e) = try_smtp_one_param(context, &param, smtp).await {
+            if let Some(at) = param.send_user.find('@') {
+                param.send_user = param.send_user.split_at(at).0.to_string();
+                try_smtp_one_param(context, &param, smtp).await?;
+            } else {
+                return Err(e);
+            }
+        }
+        Ok(param)
+    } else {
+        try_smtp_one_param(context, &param, smtp).await?;
+        Ok(param)
     }
-    try_smtp_one_param(context, param, &mut smtp).await
 }
 
 async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Smtp) -> Result<()> {
@@ -621,7 +681,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_no_panic_on_bad_credentials() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::Addr, Some("probably@unexistant.addr"))
             .await
@@ -635,7 +695,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_offline_autoconfig() {
-        let context = dummy_context().await.ctx;
+        let context = TestContext::new().await.ctx;
 
         let mut params = LoginParam::new();
         params.addr = "someone123@example.org".to_string();
@@ -646,5 +706,14 @@ mod tests {
         let found_params = get_offline_autoconfig(&context, &params).unwrap();
         assert_eq!(found_params.mail_server, "imap.nauta.cu".to_string());
         assert_eq!(found_params.send_server, "smtp.nauta.cu".to_string());
+
+        assert_eq!(
+            found_params.imap_certificate_checks,
+            CertificateChecks::Automatic
+        );
+        assert_eq!(
+            found_params.smtp_certificate_checks,
+            CertificateChecks::Automatic
+        );
     }
 }

src/constants.rs

@@ -84,6 +84,20 @@ impl Default for KeyGenType {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(i8)]
+pub enum VideochatType {
+    Unknown = 0,
+    BasicWebrtc = 1,
+    Jitsi = 2,
+}
+
+impl Default for VideochatType {
+    fn default() -> Self {
+        VideochatType::Unknown
+    }
+}
+
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
@@ -296,6 +310,9 @@ pub enum Viewtype {
     /// The file is set via dc_msg_set_file()
     /// and retrieved via dc_msg_get_file().
     File = 60,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation = 70,
 }
 
 impl Default for Viewtype {
@@ -323,54 +340,6 @@ const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
 const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
 const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
 const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
-
-// TODO: Strings need some doumentation about used placeholders.
-// These constants are used to set stock translation strings
-
-const DC_STR_NOMESSAGES: usize = 1;
-const DC_STR_SELF: usize = 2;
-const DC_STR_DRAFT: usize = 3;
-const DC_STR_VOICEMESSAGE: usize = 7;
-const DC_STR_DEADDROP: usize = 8;
-const DC_STR_IMAGE: usize = 9;
-const DC_STR_VIDEO: usize = 10;
-const DC_STR_AUDIO: usize = 11;
-const DC_STR_FILE: usize = 12;
-const DC_STR_STATUSLINE: usize = 13;
-const DC_STR_NEWGROUPDRAFT: usize = 14;
-const DC_STR_MSGGRPNAME: usize = 15;
-const DC_STR_MSGGRPIMGCHANGED: usize = 16;
-const DC_STR_MSGADDMEMBER: usize = 17;
-const DC_STR_MSGDELMEMBER: usize = 18;
-const DC_STR_MSGGROUPLEFT: usize = 19;
-const DC_STR_GIF: usize = 23;
-const DC_STR_ENCRYPTEDMSG: usize = 24;
-const DC_STR_E2E_AVAILABLE: usize = 25;
-const DC_STR_ENCR_TRANSP: usize = 27;
-const DC_STR_ENCR_NONE: usize = 28;
-const DC_STR_CANTDECRYPT_MSG_BODY: usize = 29;
-const DC_STR_FINGERPRINTS: usize = 30;
-const DC_STR_READRCPT: usize = 31;
-const DC_STR_READRCPT_MAILBODY: usize = 32;
-const DC_STR_MSGGRPIMGDELETED: usize = 33;
-const DC_STR_E2E_PREFERRED: usize = 34;
-const DC_STR_CONTACT_VERIFIED: usize = 35;
-const DC_STR_CONTACT_NOT_VERIFIED: usize = 36;
-const DC_STR_CONTACT_SETUP_CHANGED: usize = 37;
-const DC_STR_ARCHIVEDCHATS: usize = 40;
-const DC_STR_STARREDMSGS: usize = 41;
-const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
-const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_CANNOT_LOGIN: usize = 60;
-const DC_STR_SERVER_RESPONSE: usize = 61;
-const DC_STR_MSGACTIONBYUSER: usize = 62;
-const DC_STR_MSGACTIONBYME: usize = 63;
-const DC_STR_MSGLOCATIONENABLED: usize = 64;
-const DC_STR_MSGLOCATIONDISABLED: usize = 65;
-const DC_STR_LOCATION: usize = 66;
-const DC_STR_STICKER: usize = 67;
-const DC_STR_COUNT: usize = 67;
 
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 

src/contact.rs

@@ -13,7 +13,7 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
@@ -245,7 +245,7 @@ impl Contact {
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
-        context.emit_event(Event::ContactsChanged(
+        context.emit_event(EventType::ContactsChanged(
             if sth_modified == Modifier::Created {
                 Some(contact_id)
             } else {
@@ -273,7 +273,7 @@ impl Contact {
             .await
             .is_ok()
         {
-            context.emit_event(Event::MsgsChanged {
+            context.emit_event(EventType::MsgsChanged {
                 chat_id: ChatId::new(0),
                 msg_id: MsgId::new(0),
             });
@@ -533,7 +533,7 @@ impl Contact {
             }
         }
         if modify_cnt > 0 {
-            context.emit_event(Event::ContactsChanged(None));
+            context.emit_event(EventType::ContactsChanged(None));
         }
 
         Ok(modify_cnt)
@@ -678,7 +678,7 @@ impl Contact {
         let mut ret = String::new();
 
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let peerstate = Peerstate::from_addr(context, &contact.addr).await;
+            let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
             let loginparam = LoginParam::from_database(context, "configured_").await;
 
             if peerstate.is_some()
@@ -784,7 +784,7 @@ impl Contact {
                 .await
             {
                 Ok(_) => {
-                    context.emit_event(Event::ContactsChanged(None));
+                    context.emit_event(EventType::ContactsChanged(None));
                     return Ok(());
                 }
                 Err(err) => {
@@ -939,7 +939,17 @@ impl Contact {
             }
         }
 
-        let peerstate = Peerstate::from_addr(context, &self.addr).await;
+        let peerstate = match Peerstate::from_addr(context, &self.addr).await {
+            Ok(peerstate) => peerstate,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Failed to load peerstate for address {}: {}", self.addr, err
+                );
+                return VerifiedStatus::Unverified;
+            }
+        };
+
         if let Some(ps) = peerstate {
             if ps.verified_key.is_some() {
                 return VerifiedStatus::BidirectVerified;
@@ -1029,10 +1039,10 @@ pub fn addr_normalize(addr: &str) -> &str {
     let norm = addr.trim();
 
     if norm.starts_with("mailto:") {
-        return &norm[7..];
+        norm.get(7..).unwrap_or(norm)
+    } else {
+        norm
     }
-
-    norm
 }
 
 fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
@@ -1042,11 +1052,15 @@ fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (Stri
     if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.as_ref().is_empty() {
-                normalize_name(&captures[1])
+                captures
+                    .get(1)
+                    .map_or("".to_string(), |m| normalize_name(m.as_str()))
             } else {
                 name.as_ref().to_string()
             },
-            captures[2].to_string(),
+            captures
+                .get(2)
+                .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
         (name.as_ref().to_string(), addr.as_ref().to_string())
@@ -1079,33 +1093,57 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
                     paramsv![new_blocking, 100, contact_id as i32],
                 ).await.is_ok() {
                     Contact::mark_noticed(context, contact_id).await;
-                    context.emit_event(Event::ContactsChanged(None));
+                    context.emit_event(EventType::ContactsChanged(None));
                 }
         }
     }
 }
 
+/// Set profile image for a contact.
+///
+/// The given profile image is expected to be already in the blob directory
+/// as profile images can be set only by receiving messages, this should be always the case, however.
+///
+/// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
+/// this typically happens if we see message with our own profile image, sent from another device.
 pub(crate) async fn set_profile_image(
     context: &Context,
     contact_id: u32,
     profile_image: &AvatarAction,
+    was_encrypted: bool,
 ) -> Result<()> {
-    // the given profile image is expected to be already in the blob directory
-    // as profile images can be set only by receiving messages, this should be always the case, however.
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            contact.param.set(Param::ProfileImage, profile_image);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context
+                        .set_config(Config::Selfavatar, Some(profile_image))
+                        .await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar.");
+                }
+            } else {
+                contact.param.set(Param::ProfileImage, profile_image);
+            }
             true
         }
         AvatarAction::Delete => {
-            contact.param.remove(Param::ProfileImage);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context.set_config(Config::Selfavatar, None).await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar deletion.");
+                }
+            } else {
+                contact.param.remove(Param::ProfileImage);
+            }
             true
         }
     };
     if changed {
         contact.update_param(context).await?;
-        context.emit_event(Event::ContactsChanged(Some(contact_id)));
+        context.emit_event(EventType::ContactsChanged(Some(contact_id)));
     }
     Ok(())
 }
@@ -1113,38 +1151,21 @@ pub(crate) async fn set_profile_image(
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
-/// - Convert names as "Petersen, Björn" to "Björn Petersen"
 /// - Trims the resulting string
 ///
 /// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
 pub fn normalize_name(full_name: impl AsRef<str>) -> String {
-    let mut full_name = full_name.as_ref().trim();
+    let full_name = full_name.as_ref().trim();
     if full_name.is_empty() {
         return full_name.into();
     }
 
-    let len = full_name.len();
-    if len > 1 {
-        let firstchar = full_name.as_bytes()[0];
-        let lastchar = full_name.as_bytes()[len - 1];
-        if firstchar == b'\'' && lastchar == b'\''
-            || firstchar == b'\"' && lastchar == b'\"'
-            || firstchar == b'<' && lastchar == b'>'
-        {
-            full_name = &full_name[1..len - 1];
-        }
+    match full_name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => full_name
+            .get(1..full_name.len() - 1)
+            .map_or("".to_string(), |s| s.trim().into()),
+        _ => full_name.to_string(),
     }
-
-    if let Some(p1) = full_name.find(',') {
-        let (last_name, first_name) = full_name.split_at(p1);
-
-        let last_name = last_name.trim();
-        let first_name = (&first_name[1..]).trim();
-
-        return format!("{} {}", first_name, last_name);
-    }
-
-    full_name.trim().into()
 }
 
 fn cat_fingerprint(
@@ -1235,7 +1256,6 @@ mod tests {
 
     #[test]
     fn test_normalize_name() {
-        assert_eq!(&normalize_name("Doe, John"), "John Doe");
         assert_eq!(&normalize_name(" hello world   "), "hello world");
         assert_eq!(&normalize_name("<"), "<");
         assert_eq!(&normalize_name(">"), ">");
@@ -1270,7 +1290,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_contacts() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let contacts = Contact::get_all(&context.ctx, 0, Some("some2"))
             .await
             .unwrap();
@@ -1294,10 +1314,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_self_addr() -> Result<()> {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         assert!(t.ctx.is_self_addr("me@me.org").await.is_err());
 
-        let addr = configure_alice_keypair(&t.ctx).await;
+        let addr = t.configure_alice().await;
         assert_eq!(t.ctx.is_self_addr("me@me.org").await?, false);
         assert_eq!(t.ctx.is_self_addr(&addr).await?, true);
 
@@ -1307,7 +1327,7 @@ mod tests {
     #[async_std::test]
     async fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let book = concat!(
             "  Name one  \n one@eins.org \n",
             "Name two\ntwo@deux.net\n",
@@ -1400,10 +1420,10 @@ mod tests {
         assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
-        assert_eq!(contact.get_name(), "Alice Wonderland");
-        assert_eq!(contact.get_display_name(), "Alice Wonderland");
+        assert_eq!(contact.get_name(), "Wonderland, Alice");
+        assert_eq!(contact.get_display_name(), "Wonderland, Alice");
         assert_eq!(contact.get_addr(), "alice@w.de");
-        assert_eq!(contact.get_name_n_addr(), "Alice Wonderland (alice@w.de)");
+        assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
 
         // check SELF
         let contact = Contact::load_from_db(&t.ctx, DC_CONTACT_ID_SELF)
@@ -1420,7 +1440,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
@@ -1483,7 +1503,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_create_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t.ctx, "", "claire@example.org")
@@ -1530,7 +1550,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_edit_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "dave@example.org"
         let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org")
@@ -1574,7 +1594,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_name_in_address() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let contact_id = Contact::create(&t.ctx, "", "<dave@example.org>")
             .await
@@ -1587,7 +1607,7 @@ mod tests {
             .await
             .unwrap();
         let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
-        assert_eq!(contact.get_name(), "Dave Mueller");
+        assert_eq!(contact.get_name(), "Mueller, Dave");
         assert_eq!(contact.get_addr(), "dave@example.org");
 
         let contact_id = Contact::create(&t.ctx, "name1", "name2 <dave@example.org>")

src/context.rs

@@ -6,6 +6,7 @@ use std::ops::Deref;
 
 use async_std::path::{Path, PathBuf};
 use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use async_std::task;
 
 use crate::chat::*;
 use crate::config::Config;
@@ -13,13 +14,11 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::dc_tools::duration_to_str;
 use crate::error::*;
-use crate::events::{Event, EventEmitter, Events};
-use crate::job::{self, Action};
+use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message, MessengerMessage, MsgId};
-use crate::param::Params;
+use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
 use crate::sql::Sql;
 use std::time::SystemTime;
@@ -52,10 +51,16 @@ pub struct InnerContext {
     pub(crate) generating_key_mutex: Mutex<()>,
     /// Mutex to enforce only a single running oauth2 is running.
     pub(crate) oauth2_mutex: Mutex<()>,
+    /// Mutex to prevent a race condition when a "your pw is wrong" warning is sent, resulting in multiple messeges being sent.
+    pub(crate) wrong_pw_warning_mutex: Mutex<()>,
     pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
+
+    /// Id for this context on the current device.
+    pub(crate) id: u32,
 
     creation_time: SystemTime,
 }
@@ -84,7 +89,7 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context.
-    pub async fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
+    pub async fn new(os_name: String, dbfile: PathBuf, id: u32) -> Result<Context> {
         // pretty_env_logger::try_init_timed().ok();
 
         let mut blob_fname = OsString::new();
@@ -94,13 +99,14 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(os_name, dbfile, blobdir).await
+        Context::with_blobdir(os_name, dbfile, blobdir, id).await
     }
 
-    pub async fn with_blobdir(
+    pub(crate) async fn with_blobdir(
         os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
+        id: u32,
     ) -> Result<Context> {
         ensure!(
             blobdir.is_dir().await,
@@ -109,6 +115,7 @@ impl Context {
         );
 
         let inner = InnerContext {
+            id,
             blobdir,
             dbfile,
             os_name: Some(os_name),
@@ -118,9 +125,11 @@ impl Context {
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
+            wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
         };
 
@@ -184,8 +193,11 @@ impl Context {
     }
 
     /// Emits a single event.
-    pub fn emit_event(&self, event: Event) {
-        self.events.emit(event);
+    pub fn emit_event(&self, event: EventType) {
+        self.events.emit(Event {
+            id: self.id,
+            typ: event,
+        });
     }
 
     /// Get the next queued event.
@@ -193,6 +205,11 @@ impl Context {
         self.events.get_emitter()
     }
 
+    /// Get the ID of this context.
+    pub fn get_id(&self) -> u32 {
+        self.id
+    }
+
     // Ongoing process allocation/free/check
 
     pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
@@ -455,32 +472,11 @@ impl Context {
             == Some(folder_name.as_ref().to_string())
     }
 
-    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove).await {
-            return;
-        }
-
-        if self.is_mvbox(folder).await {
-            return;
-        }
-        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
-            if msg.is_setupmessage() {
-                // do not move setup messages;
-                // there may be a non-delta device that wants to handle it
-                return;
-            }
-
-            match msg.is_dc_message {
-                MessengerMessage::No => {}
-                MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job::add(
-                        self,
-                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
-                    )
-                    .await;
-                }
-            }
-        }
+    pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+        let mut blob_fname = OsString::new();
+        blob_fname.push(dbfile.file_name().unwrap_or_default());
+        blob_fname.push("-blobs");
+        dbfile.with_file_name(blob_fname)
     }
 }
 
@@ -534,13 +530,13 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new("FakeOs".into(), dbfile.into()).await;
+        let res = Context::new("FakeOs".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
     #[async_std::test]
     async fn test_get_fresh_msgs() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let fresh = t.ctx.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
@@ -549,7 +545,9 @@ mod tests {
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -560,7 +558,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new("FakeOS".into(), dbfile.into()).await;
+        let res = Context::new("FakeOS".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -570,7 +568,9 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -580,7 +580,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -589,19 +589,19 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         std::mem::drop(t.ctx);
     }
 
     #[async_std::test]
     async fn test_get_info() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let info = t.ctx.get_info().await;
         assert!(info.get("database_dir").is_some());

src/dc_receive_imf.rs

@@ -10,8 +10,9 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer};
 use crate::error::{bail, ensure, format_err, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
 use crate::message::{self, MessageState, MessengerMessage, MsgId};
@@ -45,13 +46,14 @@ pub async fn dc_receive_imf(
 ) -> Result<()> {
     info!(
         context,
-        "Receiving message {}/{}...",
+        "Receiving message {}/{}, seen={}...",
         if !server_folder.as_ref().is_empty() {
             server_folder.as_ref()
         } else {
             "?"
         },
         server_uid,
+        seen
     );
 
     if std::env::var(crate::DCC_MIME_DEBUG).unwrap_or_default() == "2" {
@@ -91,8 +93,8 @@ pub async fn dc_receive_imf(
         if let Some(create_event_to_send) = create_event_to_send {
             for (chat_id, msg_id) in created_db_entries {
                 let event = match create_event_to_send {
-                    CreateEvent::MsgsChanged => Event::MsgsChanged { msg_id, chat_id },
-                    CreateEvent::IncomingMsg => Event::IncomingMsg { msg_id, chat_id },
+                    CreateEvent::MsgsChanged => EventType::MsgsChanged { msg_id, chat_id },
+                    CreateEvent::IncomingMsg => EventType::IncomingMsg { msg_id, chat_id },
                 };
                 context.emit_event(event);
             }
@@ -195,9 +197,16 @@ pub async fn dc_receive_imf(
     }
 
     if let Some(avatar_action) = &mime_parser.user_avatar {
-        match contact::set_profile_image(&context, from_id, avatar_action).await {
+        match contact::set_profile_image(
+            &context,
+            from_id,
+            avatar_action,
+            mime_parser.was_encrypted(),
+        )
+        .await
+        {
             Ok(()) => {
-                context.emit_event(Event::ChatModified(chat_id));
+                context.emit_event(EventType::ChatModified(chat_id));
             }
             Err(err) => {
                 warn!(context, "reveive_imf cannot update profile image: {}", err);
@@ -222,11 +231,29 @@ pub async fn dc_receive_imf(
                 )
                 .await;
             }
-        } else {
+        } else if insert_msg_id
+            .needs_move(context, server_folder.as_ref())
+            .await
+            .unwrap_or_default()
+        {
             // Move message if we don't delete it immediately.
-            context
-                .do_heuristics_moves(server_folder.as_ref(), insert_msg_id)
-                .await;
+            job::add(
+                context,
+                job::Job::new(Action::MoveMsg, insert_msg_id.to_u32(), Params::new(), 0),
+            )
+            .await;
+        } else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
+            // This is a Delta Chat MDN. Mark as read.
+            job::add(
+                context,
+                job::Job::new(
+                    Action::MarkseenMsgOnImap,
+                    insert_msg_id.to_u32(),
+                    Params::new(),
+                    0,
+                ),
+            )
+            .await;
         }
     }
 
@@ -238,7 +265,7 @@ pub async fn dc_receive_imf(
     cleanup(context, &create_event_to_send, created_db_entries);
 
     mime_parser
-        .handle_reports(context, from_id, sent_timestamp)
+        .handle_reports(context, from_id, sent_timestamp, &mime_parser.parts)
         .await;
 
     Ok(())
@@ -332,7 +359,7 @@ async fn add_parts(
         return Ok(());
     }
 
-    let mut msgrmsg = if mime_parser.has_chat_version() {
+    let mut is_dc_message = if mime_parser.has_chat_version() {
         MessengerMessage::Yes
     } else if is_reply_to_messenger_message(context, mime_parser).await {
         MessengerMessage::Reply
@@ -344,7 +371,7 @@ async fn add_parts(
     let show_emails =
         ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await).unwrap_or_default();
     if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage
-        && msgrmsg == MessengerMessage::No
+        && is_dc_message == MessengerMessage::No
     {
         // this message is a classic email not a chat-message nor a reply to one
         match show_emails {
@@ -373,7 +400,7 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get(HeaderDef::SecureJoin).is_some() {
-            msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting
+            is_dc_message = MessengerMessage::Yes; // avoid discarding by show_emails setting
             *chat_id = ChatId::new(0);
             allow_creation = true;
             match handle_securejoin_handshake(context, mime_parser, from_id).await {
@@ -405,6 +432,14 @@ async fn add_parts(
                 .await
                 .unwrap_or_default();
 
+        if chat_id.is_unset() && mime_parser.failure_report.is_some() {
+            *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+            info!(
+                context,
+                "Message belongs to an NDN and is not shown in a chat.",
+            );
+        }
+
         // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
         // it might also be blocked and displayed in the deaddrop as a result
         if chat_id.is_unset() {
@@ -498,7 +533,7 @@ async fn add_parts(
         if Blocked::Not != chat_id_blocked
             && state == MessageState::InFresh
             && !incoming_origin.is_known()
-            && msgrmsg == MessengerMessage::No
+            && is_dc_message == MessengerMessage::No
             && show_emails != ShowEmails::All
         {
             state = MessageState::InNoticed;
@@ -513,7 +548,7 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get(HeaderDef::SecureJoin).is_some() {
-            msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting
+            is_dc_message = MessengerMessage::Yes; // avoid discarding by show_emails setting
             *chat_id = ChatId::new(0);
             allow_creation = true;
             match observe_securejoin_on_other_device(context, mime_parser, to_id).await {
@@ -552,7 +587,7 @@ async fn add_parts(
                 }
             }
             if chat_id.is_unset() && allow_creation {
-                let create_blocked = if MessengerMessage::No != msgrmsg
+                let create_blocked = if MessengerMessage::No != is_dc_message
                     && !Contact::is_blocked_load(context, to_id).await
                 {
                     Blocked::Not
@@ -597,10 +632,75 @@ async fn add_parts(
             *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
         }
     }
+
+    // Extract ephemeral timer from the message.
+    let mut ephemeral_timer = if let Some(value) = mime_parser.get(HeaderDef::EphemeralTimer) {
+        match value.parse::<EphemeralTimer>() {
+            Ok(timer) => timer,
+            Err(err) => {
+                warn!(
+                    context,
+                    "can't parse ephemeral timer \"{}\": {}", value, err
+                );
+                EphemeralTimer::Disabled
+            }
+        }
+    } else {
+        EphemeralTimer::Disabled
+    };
+
+    let location_kml_is = mime_parser.location_kml.is_some();
+    let is_mdn = !mime_parser.mdn_reports.is_empty();
+
+    // Apply ephemeral timer changes to the chat.
+    //
+    // Only non-hidden timers are applied now. Timers from hidden
+    // messages such as read receipts can be useful to detect
+    // ephemeral timer support, but timer changes without visible
+    // received messages may be confusing to the user.
+    if !*hidden
+        && !location_kml_is
+        && !is_mdn
+        && (*chat_id).get_ephemeral_timer(context).await? != ephemeral_timer
+    {
+        if let Err(err) = (*chat_id)
+            .inner_set_ephemeral_timer(context, ephemeral_timer)
+            .await
+        {
+            warn!(
+                context,
+                "failed to modify timer for chat {}: {}", chat_id, err
+            );
+        } else if mime_parser.is_system_message != SystemMessage::EphemeralTimerChanged {
+            chat::add_info_msg(
+                context,
+                *chat_id,
+                stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
+            )
+            .await;
+        }
+    }
+
+    if mime_parser.is_system_message == SystemMessage::EphemeralTimerChanged {
+        set_better_msg(
+            mime_parser,
+            stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
+        );
+
+        // Do not delete the system message itself.
+        //
+        // This prevents confusion when timer is changed
+        // to 1 week, and then changed to 1 hour: after 1
+        // hour, only the message about the change to 1
+        // week is left.
+        ephemeral_timer = EphemeralTimer::Disabled;
+    }
+
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
+    let in_fresh = state == MessageState::InFresh;
     let rcvd_timestamp = time();
-    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, !seen).await;
+    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, in_fresh).await;
     *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
@@ -627,7 +727,6 @@ async fn add_parts(
 
     let mut parts = std::mem::replace(&mut mime_parser.parts, Vec::new());
     let server_folder = server_folder.as_ref().to_string();
-    let location_kml_is = mime_parser.location_kml.is_some();
     let is_system_message = mime_parser.is_system_message;
     let mime_headers = if save_mime_headers {
         Some(String::from_utf8_lossy(imf_raw).to_string())
@@ -637,7 +736,6 @@ async fn add_parts(
     let sent_timestamp = *sent_timestamp;
     let is_hidden = *hidden;
     let chat_id = *chat_id;
-    let is_mdn = !mime_parser.reports.is_empty();
 
     // TODO: can this clone be avoided?
     let rfc724_mid = rfc724_mid.to_string();
@@ -654,8 +752,8 @@ async fn add_parts(
                     "INSERT INTO msgs \
          (rfc724_mid, server_folder, server_uid, chat_id, from_id, to_id, timestamp, \
          timestamp_sent, timestamp_rcvd, type, state, msgrmsg,  txt, txt_raw, param, \
-         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references) \
-         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?);",
+         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references, error, ephemeral_timer, ephemeral_timestamp) \
+         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?, ?,?,?);",
                 )?;
 
                 let is_location_kml = location_kml_is
@@ -677,6 +775,15 @@ async fn add_parts(
                     part.param.set_int(Param::Cmd, is_system_message as i32);
                 }
 
+                let ephemeral_timestamp = if in_fresh {
+                    0
+                } else {
+                    match ephemeral_timer {
+                        EphemeralTimer::Disabled => 0,
+                        EphemeralTimer::Enabled { duration } => rcvd_timestamp + i64::from(duration)
+                    }
+                };
+
                 stmt.execute(paramsv![
                     rfc724_mid,
                     server_folder,
@@ -689,7 +796,7 @@ async fn add_parts(
                     rcvd_timestamp,
                     part.typ,
                     state,
-                    msgrmsg,
+                    is_dc_message,
                     part.msg,
                     // txt_raw might contain invalid utf8
                     txt_raw,
@@ -699,6 +806,9 @@ async fn add_parts(
                     mime_headers,
                     mime_in_reply_to,
                     mime_references,
+                    part.error,
+                    ephemeral_timer,
+                    ephemeral_timestamp
                 ])?;
 
                 drop(stmt);
@@ -754,15 +864,17 @@ async fn add_parts(
         chat.update_param(context).await?;
         Ok(())
     }
-    update_last_subject(context, chat_id, mime_parser)
-        .await
-        .unwrap_or_else(|e| {
-            warn!(
-                context,
-                "Could not update LastSubject of chat: {}",
-                e.to_string()
-            )
-        });
+    if !is_mdn {
+        update_last_subject(context, chat_id, mime_parser)
+            .await
+            .unwrap_or_else(|e| {
+                warn!(
+                    context,
+                    "Could not update LastSubject of chat: {}",
+                    e.to_string()
+                )
+            });
+    }
 
     Ok(())
 }
@@ -823,7 +935,7 @@ async fn save_locations(
         }
     }
     if send_event {
-        context.emit_event(Event::LocationChanged(Some(from_id)));
+        context.emit_event(EventType::LocationChanged(Some(from_id)));
     }
 }
 
@@ -1084,23 +1196,19 @@ async fn create_or_lookup_group(
 
     // again, check chat_id
     if chat_id.is_special() {
-        return if group_explicitly_left {
-            Ok((ChatId::new(DC_CHAT_ID_TRASH), chat_id_blocked))
+        if mime_parser.decrypting_failed {
+            // It is possible that the message was sent to a valid,
+            // yet unknown group, which was rejected because
+            // Chat-Group-Name, which is in the encrypted part, was
+            // not found. We can't create a properly named group in
+            // this case, so assign error message to 1:1 chat with the
+            // sender instead.
+            return Ok((ChatId::new(0), Blocked::Not));
         } else {
-            create_or_lookup_adhoc_group(
-                context,
-                mime_parser,
-                allow_creation,
-                create_blocked,
-                from_id,
-                to_ids,
-            )
-            .await
-            .map_err(|err| {
-                warn!(context, "failed to create ad-hoc group: {:?}", err);
-                err
-            })
-        };
+            // The message was decrypted successfully, but contains a late "quit" or otherwise
+            // unwanted message.
+            return Ok((ChatId::new(DC_CHAT_ID_TRASH), chat_id_blocked));
+        }
     }
 
     // We have a valid chat_id > DC_CHAT_ID_LAST_SPECIAL.
@@ -1131,7 +1239,7 @@ async fn create_or_lookup_group(
                     .await
                     .is_ok()
                 {
-                    context.emit_event(Event::ChatModified(chat_id));
+                    context.emit_event(EventType::ChatModified(chat_id));
                 }
             }
         }
@@ -1190,7 +1298,7 @@ async fn create_or_lookup_group(
     }
 
     if send_EVENT_CHAT_MODIFIED {
-        context.emit_event(Event::ChatModified(chat_id));
+        context.emit_event(EventType::ChatModified(chat_id));
     }
     Ok((chat_id, chat_id_blocked))
 }
@@ -1285,12 +1393,10 @@ async fn create_or_lookup_adhoc_group(
         // decrypted.
         //
         // The subject may be encrypted and contain a placeholder such
-        // as "...". Besides that, it is possible that the message was
-        // sent to a valid, yet unknown group, which was rejected
-        // because Chat-Group-Name, which is in the encrypted part,
-        // was not found. Generating a new ID in this case would
-        // result in creation of a twin group with a different group
-        // ID.
+        // as "...". It can also be a COI group, with encrypted
+        // Chat-Group-ID and incompatible Message-ID format.
+        //
+        // Instead, assign the message to 1:1 chat with the sender.
         warn!(
             context,
             "not creating ad-hoc group for message that cannot be decrypted"
@@ -1329,7 +1435,7 @@ async fn create_or_lookup_adhoc_group(
         chat::add_to_chat_contacts_table(context, new_chat_id, member_id).await;
     }
 
-    context.emit_event(Event::ChatModified(new_chat_id));
+    context.emit_event(EventType::ChatModified(new_chat_id));
 
     Ok((new_chat_id, create_blocked))
 }
@@ -1422,6 +1528,7 @@ async fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
     hex_hash(&members)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn hex_hash(s: impl AsRef<str>) -> String {
     let bytes = s.as_ref().as_bytes();
     let result = Sha256::digest(bytes);
@@ -1474,7 +1581,7 @@ async fn search_chat_ids_by_contact_ids(
                             matches = 0;
                             mismatches = 0;
                         }
-                        if matches < contact_ids.len() && contact_id == contact_ids[matches] {
+                        if contact_ids.get(matches) == Some(&contact_id) {
                             matches += 1;
                         } else {
                             mismatches += 1;
@@ -1508,7 +1615,7 @@ async fn check_verified_properties(
     // this check is skipped for SELF as there is no proper SELF-peerstate
     // and results in group-splits otherwise.
     if from_id != DC_CONTACT_ID_SELF {
-        let peerstate = Peerstate::from_addr(context, contact.get_addr()).await;
+        let peerstate = Peerstate::from_addr(context, contact.get_addr()).await?;
 
         if peerstate.is_none()
             || contact.is_verified_ex(context, peerstate.as_ref()).await
@@ -1562,7 +1669,7 @@ async fn check_verified_properties(
             context.is_self_addr(&to_addr).await
         );
         let mut is_verified = _is_verified != 0;
-        let peerstate = Peerstate::from_addr(context, &to_addr).await;
+        let peerstate = Peerstate::from_addr(context, &to_addr).await?;
 
         // mark gossiped keys (if any) as verified
         if mimeparser.gossipped_addr.contains(&to_addr) {
@@ -1602,10 +1709,11 @@ async fn check_verified_properties(
 
 fn set_better_msg(mime_parser: &mut MimeMessage, better_msg: impl AsRef<str>) {
     let msg = better_msg.as_ref();
-    if !msg.is_empty() && !mime_parser.parts.is_empty() {
-        let part = &mut mime_parser.parts[0];
-        if part.typ == Viewtype::Text {
-            part.msg = msg.to_string();
+    if !msg.is_empty() {
+        if let Some(part) = mime_parser.parts.get_mut(0) {
+            if part.typ == Viewtype::Text {
+                part.msg = msg.to_string();
+            }
         }
     }
 }
@@ -1764,10 +1872,10 @@ fn dc_create_incoming_rfc724_mid(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat::ChatVisibility;
+    use crate::chat::{ChatItem, ChatVisibility};
     use crate::chatlist::Chatlist;
     use crate::message::Message;
-    use crate::test_utils::{configured_offline_context, dummy_context};
+    use crate::test_utils::*;
 
     #[test]
     fn test_hex_hash() {
@@ -1779,7 +1887,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_grpid_simple() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello\n\
                     Subject: outer-subject\n\
                     In-Reply-To: <lqkjwelq123@123123>\n\
@@ -1796,7 +1904,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_grpid_from_multiple() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello\n\
                     Subject: outer-subject\n\
                     In-Reply-To: <Gr.HcxyMARjyJy.9-qweqwe@asd.net>\n\
@@ -1833,7 +1941,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_known_rfc724_mid() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("first message".to_string());
         let msg_id = chat::add_device_msg(&t.ctx, None, Some(&mut msg))
@@ -1849,7 +1957,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_msgrmsg_rfc724_mid() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some("first message".to_string());
         let msg_id = chat::add_device_msg(&t.ctx, None, Some(&mut msg))
@@ -1863,34 +1971,34 @@ mod tests {
         assert!(!is_msgrmsg_rfc724_mid(&t.ctx, "nonexistant@message.id").await);
     }
 
-    static MSGRMSG: &[u8] = b"From: Bob <bob@example.org>\n\
-                    To: alice@example.org\n\
+    static MSGRMSG: &[u8] = b"From: Bob <bob@example.com>\n\
+                    To: alice@example.com\n\
                     Chat-Version: 1.0\n\
                     Subject: Chat: hello\n\
-                    Message-ID: <Mr.1111@example.org>\n\
+                    Message-ID: <Mr.1111@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:55 +0000\n\
                     \n\
                     hello\n";
 
-    static ONETOONE_NOREPLY_MAIL: &[u8] = b"From: Bob <bob@example.org>\n\
-                    To: alice@example.org\n\
+    static ONETOONE_NOREPLY_MAIL: &[u8] = b"From: Bob <bob@example.com>\n\
+                    To: alice@example.com\n\
                     Subject: Chat: hello\n\
-                    Message-ID: <2222@example.org>\n\
+                    Message-ID: <2222@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     hello\n";
 
-    static GRP_MAIL: &[u8] = b"From: bob@example.org\n\
-                    To: alice@example.org, claire@example.org\n\
+    static GRP_MAIL: &[u8] = b"From: bob@example.com\n\
+                    To: alice@example.com, claire@example.com\n\
                     Subject: group with Alice, Bob and Claire\n\
-                    Message-ID: <3333@example.org>\n\
+                    Message-ID: <3333@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                     \n\
                     hello\n";
 
     #[async_std::test]
     async fn test_adhoc_group_show_chats_only() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         assert_eq!(t.ctx.get_config_int(Config::ShowEmails).await, 0);
 
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
@@ -1917,7 +2025,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_accepted_contact_unknown() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("1"))
             .await
@@ -1933,12 +2041,12 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_accepted_contact_known() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("1"))
             .await
             .unwrap();
-        Contact::create(&t.ctx, "Bob", "bob@example.org")
+        Contact::create(&t.ctx, "Bob", "bob@example.com")
             .await
             .unwrap();
         dc_receive_imf(&t.ctx, GRP_MAIL, "INBOX", 1, false)
@@ -1953,7 +2061,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_accepted_contact_accepted() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("1"))
             .await
@@ -1999,7 +2107,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_adhoc_group_show_all() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::ShowEmails, Some("2"))
             .await
@@ -2024,7 +2132,7 @@ mod tests {
     #[async_std::test]
     async fn test_read_receipt_and_unarchive() {
         // create alice's account
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
 
         // create one-to-one with bob, archive one-to-one
         let bob_id = Contact::create(&t.ctx, "bob", "bob@exampel.org")
@@ -2067,14 +2175,14 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             format!(
-                "From: alice@example.org\n\
-                 To: bob@example.org\n\
+                "From: alice@example.com\n\
+                 To: bob@example.com\n\
                  Subject: foo\n\
-                 Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Message-ID: <Gr.{}.12345678901@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Group-ID: {}\n\
                  Chat-Group-Name: foo\n\
-                 Chat-Disposition-Notification-To: alice@example.org\n\
+                 Chat-Disposition-Notification-To: alice@example.com\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                  \n\
                  hello\n",
@@ -2089,7 +2197,11 @@ mod tests {
         .unwrap();
         let msgs = chat::get_chat_msgs(&t.ctx, group_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
         let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
             .await
             .unwrap();
@@ -2103,12 +2215,12 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             format!(
-                "From: bob@example.org\n\
-                 To: alice@example.org\n\
+                "From: bob@example.com\n\
+                 To: alice@example.com\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
                  Chat-Version: 1.0\n\
-                 Message-ID: <Mr.12345678902@example.org>\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
                  Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
                  \n\
                  \n\
@@ -2122,9 +2234,9 @@ mod tests {
                  Content-Type: message/disposition-notification\n\
                  \n\
                  Reporting-UA: Delta Chat 1.28.0\n\
-                 Original-Recipient: rfc822;bob@example.org\n\
-                 Final-Recipient: rfc822;bob@example.org\n\
-                 Original-Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Original-Recipient: rfc822;bob@example.com\n\
+                 Final-Recipient: rfc822;bob@example.com\n\
+                 Original-Message-ID: <Gr.{}.12345678901@example.com>\n\
                  Disposition: manual-action/MDN-sent-automatically; displayed\n\
                  \n\
                  \n\
@@ -2164,7 +2276,7 @@ mod tests {
         // are very rare, however, we have to add them to the database (they go to the
         // "deaddrop" chat) to avoid a re-download from the server. See also [**]
 
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         let context = &t.ctx;
 
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
@@ -2172,9 +2284,9 @@ mod tests {
 
         dc_receive_imf(
             context,
-            b"To: bob@example.org\n\
+            b"To: bob@example.com\n\
                  Subject: foo\n\
-                 Message-ID: <3924@example.org>\n\
+                 Message-ID: <3924@example.com>\n\
                  Chat-Version: 1.0\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                  \n\
@@ -2193,7 +2305,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_escaped_from() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         let contact_id = Contact::create(&t.ctx, "foobar", "foobar@example.com")
             .await
             .unwrap();
@@ -2203,9 +2315,9 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
-                 To: alice@example.org\n\
+                 To: alice@example.com\n\
                  Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Message-ID: <asdklfjjaweofi@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Disposition-Notification-To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -2220,11 +2332,15 @@ mod tests {
                 .await
                 .unwrap()
                 .get_authname(),
-            "Фамилия Имя", // The name was "Имя, Фамилия" and ("lastname, firstname") and should be swapped to "firstname, lastname"
+            "Имя, Фамилия",
         );
         let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
         let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
             .await
             .unwrap();
@@ -2235,7 +2351,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_escaped_recipients() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         Contact::create(&t.ctx, "foobar", "foobar@example.com")
             .await
             .unwrap();
@@ -2249,10 +2365,10 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             b"From: Foobar <foobar@example.com>\n\
-                 To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.org\n\
+                 To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.com\n\
                  Cc: =?utf-8?q?=3Ch2=3E?= <carl@host.tld>\n\
                  Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Message-ID: <asdklfjjaweofi@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Disposition-Notification-To: <foobar@example.com>\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -2283,7 +2399,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_cc_to_contact() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         Contact::create(&t.ctx, "foobar", "foobar@example.com")
             .await
             .unwrap();
@@ -2301,10 +2417,10 @@ mod tests {
         dc_receive_imf(
             &t.ctx,
             b"From: Foobar <foobar@example.com>\n\
-                 To: alice@example.org\n\
+                 To: alice@example.com\n\
                  Cc: Carl <carl@host.tld>\n\
                  Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Message-ID: <asdklfjjaweofi@example.com>\n\
                  Chat-Version: 1.0\n\
                  Chat-Disposition-Notification-To: <foobar@example.com>\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -2324,4 +2440,187 @@ mod tests {
             "Carl"
         );
     }
+
+    #[async_std::test]
+    async fn test_parse_ndn_tiscali() {
+        test_parse_ndn(
+            "alice@tiscali.it",
+            "shenauithz@testrun.org",
+            "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
+            include_bytes!("../test-data/message/tiscali_ndn.eml"),
+            "",
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_testrun() {
+        test_parse_ndn(
+            "alice@testrun.org",
+            "hcksocnsofoejx@five.chat",
+            "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
+            include_bytes!("../test-data/message/testrun_ndn.eml"),
+            "Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_yahoo() {
+        test_parse_ndn(
+            "alice@yahoo.com",
+            "haeclirth.sinoenrat@yahoo.com",
+            "1680295672.3657931.1591783872936@mail.yahoo.com",
+            include_bytes!("../test-data/message/yahoo_ndn.eml"),
+            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com [...]"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_gmail() {
+        test_parse_ndn(
+            "alice@gmail.com",
+            "assidhfaaspocwaeofi@gmail.com",
+            "CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com",
+            include_bytes!("../test-data/message/gmail_ndn.eml"),
+            "Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp",
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_gmx() {
+        test_parse_ndn(
+            "alice@gmx.com",
+            "snaerituhaeirns@gmail.com",
+            "9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de",
+            include_bytes!("../test-data/message/gmx_ndn.eml"),
+            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp [...]"
+        )
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_posteo() {
+        test_parse_ndn(
+            "alice@posteo.org",
+            "hanerthaertidiuea@gmx.de",
+            "04422840-f884-3e37-5778-8192fe22d8e1@posteo.de",
+            include_bytes!("../test-data/message/posteo_ndn.eml"),
+            "Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)",
+        )
+        .await;
+    }
+
+    // ndn = Non Delivery Notification
+    async fn test_parse_ndn(
+        self_addr: &str,
+        foreign_addr: &str,
+        rfc724_mid_outgoing: &str,
+        raw_ndn: &[u8],
+        error_msg: &str,
+    ) {
+        let t = TestContext::new().await;
+        t.configure_addr(self_addr).await;
+
+        dc_receive_imf(
+            &t.ctx,
+            format!(
+                "From: {}\n\
+                To: {}\n\
+                Subject: foo\n\
+                Message-ID: <{}>\n\
+                Chat-Version: 1.0\n\
+                Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                \n\
+                hello\n",
+                self_addr, foreign_addr, rfc724_mid_outgoing
+            )
+            .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+
+        // Check that the ndn would be downloaded:
+        let headers = mailparse::parse_mail(raw_ndn).unwrap().headers;
+        assert!(
+            crate::imap::prefetch_should_download(&t.ctx, &headers, ShowEmails::Off)
+                .await
+                .unwrap()
+        );
+
+        dc_receive_imf(&t.ctx, raw_ndn, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+
+        assert_eq!(msg.state, MessageState::OutFailed);
+        assert_eq!(msg.error, error_msg);
+    }
+
+    #[async_std::test]
+    async fn test_parse_ndn_group_msg() {
+        let t = TestContext::new().await;
+        t.configure_addr("alice@gmail.com").await;
+
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@gmail.com\n\
+                 To: bob@example.com, assidhfaaspocwaeofi@gmail.com\n\
+                 Subject: foo\n\
+                 Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde\n\
+                 Chat-Group-Name: foo\n\
+                 Chat-Disposition-Notification-To: alice@example.com\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+
+        let raw = include_bytes!("../test-data/message/gmail_ndn_group.eml");
+        dc_receive_imf(&t.ctx, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+
+        assert_eq!(msg.state, MessageState::OutFailed);
+
+        let msgs = chat::get_chat_msgs(&t.ctx, msg.chat_id, 0, None).await;
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
+        let last_msg = Message::load_from_db(&t.ctx, *msg_id).await.unwrap();
+
+        assert_eq!(
+            last_msg.text,
+            Some(
+                t.ctx
+                    .stock_string_repl_str(
+                        StockMessage::FailedSendingTo,
+                        "assidhfaaspocwaeofi@gmail.com",
+                    )
+                    .await,
+            )
+        );
+        assert_eq!(last_msg.from_id, DC_CONTACT_ID_INFO);
+    }
 }

src/dc_tools.rs

@@ -4,6 +4,7 @@
 use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
+use std::io::Cursor;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
@@ -14,7 +15,7 @@ use rand::{thread_rng, Rng};
 
 use crate::context::Context;
 use crate::error::{bail, Error};
-use crate::events::Event;
+use crate::events::EventType;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
     0 != v && 0 == v & (v - 1)
@@ -22,6 +23,7 @@ pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let ellipse = "[...]";
 
@@ -54,6 +56,7 @@ const COLORS: [u32; 16] = [
     0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
 ];
 
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let str_lower = s.as_ref().to_lowercase();
     let mut checksum = 0;
@@ -198,7 +201,7 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
 pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str) -> String {
     let hostname = from_addr
         .find('@')
-        .map(|k| &from_addr[k..])
+        .and_then(|k| from_addr.get(k..))
         .unwrap_or("@nohost");
     match grpid {
         Some(grpid) => format!("Gr.{}.{}{}", grpid, dc_create_id(), hostname),
@@ -240,9 +243,9 @@ pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
 
 /// Returns the `(width, height)` of the given image buffer.
 pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
-    let meta = image_meta::load_from_buf(buf)?;
-
-    Ok((meta.dimensions.width, meta.dimensions.height))
+    let image = image::io::Reader::new(Cursor::new(buf)).with_guessed_format()?;
+    let dimensions = image.into_dimensions()?;
+    Ok(dimensions)
 }
 
 /// Expand paths relative to $BLOBDIR into absolute paths.
@@ -283,7 +286,7 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     let dpath = format!("{}", path.as_ref().to_string_lossy());
     match fs::remove_file(path_abs).await {
         Ok(_) => {
-            context.emit_event(Event::DeletedBlobFile(dpath));
+            context.emit_event(EventType::DeletedBlobFile(dpath));
             true
         }
         Err(err) => {
@@ -447,7 +450,7 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn dc_get_next_backup_path(
+pub(crate) async fn get_next_backup_path_old(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -467,6 +470,32 @@ pub(crate) async fn dc_get_next_backup_path(
     bail!("could not create backup file, disk full?");
 }
 
+/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+pub(crate) async fn get_next_backup_path_new(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf), Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
@@ -774,7 +803,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_file_handling() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let context = &t.ctx;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
@@ -853,7 +882,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_ne!(
             dc_create_smeared_timestamp(&t.ctx).await,
             dc_create_smeared_timestamp(&t.ctx).await
@@ -869,7 +898,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamps() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
         let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
         let next = dc_smeared_time(&t.ctx).await;
@@ -913,4 +942,22 @@ mod tests {
             "3h 1m 0s"
         );
     }
+
+    #[test]
+    fn test_get_filemeta() {
+        let data = include_bytes!("../test-data/image/avatar900x900.png");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 900);
+        assert_eq!(h, 900);
+
+        let data = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 1000);
+        assert_eq!(h, 1000);
+
+        let data = include_bytes!("../test-data/image/image100x50.gif");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 100);
+        assert_eq!(h, 50);
+    }
 }

src/dehtml.rs

@@ -46,6 +46,12 @@ pub fn dehtml(buf: &str) -> String {
             Ok(quick_xml::events::Event::End(ref e)) => dehtml_endtag_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::Text(ref e)) => dehtml_text_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::CData(ref e)) => dehtml_cdata_cb(e, &mut dehtml),
+            Ok(quick_xml::events::Event::Empty(ref e)) => {
+                // Handle empty tags as a start tag immediately followed by end tag.
+                // For example, `<p/>` is treated as `<p></p>`.
+                dehtml_starttag_cb(e, &mut dehtml, &reader);
+                dehtml_endtag_cb(&BytesEnd::borrowed(e.name()), &mut dehtml);
+            }
             Err(e) => {
                 eprintln!(
                     "Parse html error: Error at position {}: {:?}",
@@ -182,6 +188,10 @@ mod tests {
             ("&amp; bar", "& bar"),
             // Note missing '
             ("<a href='/foo.png>Hi</a> ", ""),
+            (
+                "<a href='https://get.delta.chat/'/>",
+                "[](https://get.delta.chat/)",
+            ),
             ("", ""),
         ];
         for (input, output) in cases {
@@ -191,10 +201,10 @@ mod tests {
 
     #[test]
     fn test_dehtml_parse_br() {
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2<br/>line3\n\r";
         let plain = dehtml(html);
 
-        assert_eq!(plain, "line1\n\r\r\rline2");
+        assert_eq!(plain, "line1\n\r\r\rline2\nline3");
     }
 
     #[test]

src/e2ee.rs

@@ -15,7 +15,6 @@ use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
-use crate::securejoin::handle_degrade_event;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
@@ -115,6 +114,14 @@ impl EncryptHelper {
     }
 }
 
+/// Tries to decrypt a message, but only if it is structured as an
+/// Autocrypt message.
+///
+/// Returns decrypted body and a set of valid signature fingerprints
+/// if successful.
+///
+/// If the message is wrongly signed, this will still return the decrypted
+/// message but the HashSet will be empty.
 pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
@@ -132,7 +139,7 @@ pub async fn try_decrypt(
     let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
 
     if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &from).await;
+        peerstate = Peerstate::from_addr(context, &from).await?;
 
         if let Some(ref mut peerstate) = peerstate {
             if let Some(ref header) = autocryptheader {
@@ -155,17 +162,14 @@ pub async fn try_decrypt(
     let mut signatures = HashSet::default();
 
     if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-        peerstate = Peerstate::from_addr(&context, &from).await;
+        peerstate = Peerstate::from_addr(&context, &from).await?;
     }
     if let Some(peerstate) = peerstate {
-        if peerstate.degrade_event.is_some() {
-            handle_degrade_event(context, &peerstate).await?;
-        }
-        if let Some(key) = peerstate.gossip_key {
-            public_keyring_for_validate.add(key);
-        }
+        peerstate.handle_fingerprint_change(context).await?;
         if let Some(key) = peerstate.public_key {
             public_keyring_for_validate.add(key);
+        } else if let Some(key) = peerstate.gossip_key {
+            public_keyring_for_validate.add(key);
         }
     }
 
@@ -187,24 +191,23 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
         "Not a multipart/encrypted message: {}",
         mail.ctype.mimetype
     );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
+    if let [first_part, second_part] = &mail.subparts[..] {
+        ensure!(
+            first_part.ctype.mimetype == "application/pgp-encrypted",
+            "Invalid Autocrypt Level 1 version part: {:?}",
+            first_part.ctype,
+        );
 
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
+        ensure!(
+            second_part.ctype.mimetype == "application/octet-stream",
+            "Invalid Autocrypt Level 1 encrypted part: {:?}",
+            second_part.ctype
+        );
 
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
-    );
-
-    Ok(&mail.subparts[1])
+        Ok(second_part)
+    } else {
+        bail!("Invalid Autocrypt Level 1 Mime Parts")
+    }
 }
 
 async fn decrypt_if_autocrypt_message<'a>(
@@ -214,13 +217,6 @@ async fn decrypt_if_autocrypt_message<'a>(
     public_keyring_for_validate: Keyring<SignedPublicKey>,
     ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
-    //  The returned bool is true if we detected an Autocrypt-encrypted
-    // message and successfully decrypted it. Decryption then modifies the
-    // passed in mime structure in place. The returned bool is false
-    // if it was not an Autocrypt message.
-    //
-    // Errors are returned for failures related to decryption of AC-messages.
-
     let encrypted_data_part = match get_autocrypt_mime(mail) {
         Err(_) => {
             // not an autocrypt mime message, abort and ignore
@@ -260,13 +256,16 @@ async fn decrypt_part(
         )
         .await?;
 
-        ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
+        // If the message was wrongly or not signed, still return the plain text.
+        // The caller has to check the signatures then.
+
         return Ok(Some(plain));
     }
 
     Ok(None)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     if let Some(index) = input.iter().position(|b| *b > b' ') {
         if input.len() - index > 26 {
@@ -327,14 +326,14 @@ mod tests {
 
         #[async_std::test]
         async fn test_prexisting() {
-            let t = dummy_context().await;
-            let test_addr = configure_alice_keypair(&t.ctx).await;
+            let t = TestContext::new().await;
+            let test_addr = t.configure_alice().await;
             assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
-            let t = dummy_context().await;
+            let t = TestContext::new().await;
             assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
         }
     }

src/ephemeral.rs

@@ -0,0 +1,528 @@
+//! # Ephemeral messages
+//!
+//! Ephemeral messages are messages that have an Ephemeral-Timer
+//! header attached to them, which specifies time in seconds after
+//! which the message should be deleted both from the device and from
+//! the server. The timer is started when the message is marked as
+//! seen, which usually happens when its contents is displayed on
+//! device screen.
+//!
+//! Each chat, including 1:1, group chats and "saved messages" chat,
+//! has its own ephemeral timer setting, which is applied to all
+//! messages sent to the chat. The setting is synchronized to all the
+//! devices participating in the chat by applying the timer value from
+//! all received messages, including BCC-self ones, to the chat. This
+//! way the setting is eventually synchronized among all participants.
+//!
+//! When user changes ephemeral timer setting for the chat, a system
+//! message is automatically sent to update the setting for all
+//! participants. This allows changing the setting for a chat like any
+//! group chat setting, e.g. name and avatar, without the need to
+//! write an actual message.
+//!
+//! ## Device settings
+//!
+//! In addition to per-chat ephemeral message setting, each device has
+//! two global user-configured settings that complement per-chat
+//! settings: `delete_device_after` and `delete_server_after`. These
+//! settings are not synchronized among devices and apply to all
+//! messages known to the device, including messages sent or received
+//! before configuring the setting.
+//!
+//! `delete_device_after` configures the maximum time device is
+//! storing the messages locally. `delete_server_after` configures the
+//! time after which device will delete the messages it knows about
+//! from the server.
+//!
+//! ## How messages are deleted
+//!
+//! When the message is deleted locally, its contents is removed and
+//! it is moved to the trash chat. This database entry is then used to
+//! track the Message-ID and corresponding IMAP folder and UID until
+//! the message is deleted from the server. Vice versa, when device
+//! deletes the message from the server, it removes IMAP folder and
+//! UID information, but keeps the message contents. When database
+//! entry is both moved to trash chat and does not contain UID
+//! information, it is deleted from the database, leaving no trace of
+//! the message.
+//!
+//! ## When messages are deleted
+//!
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
+//!
+//! Server deletion happens by generating IMAP deletion jobs based on
+//! the database entries which are expired either according to their
+//! ephemeral message timers or global `delete_server_after` setting.
+
+use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use crate::constants::{
+    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::error::{ensure, Error};
+use crate::events::EventType;
+use crate::message::{Message, MessageState, MsgId};
+use crate::mimeparser::SystemMessage;
+use crate::sql;
+use crate::stock::StockMessage;
+use async_std::task;
+use serde::{Deserialize, Serialize};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
+pub enum Timer {
+    Disabled,
+    Enabled { duration: u32 },
+}
+
+impl Timer {
+    pub fn to_u32(self) -> u32 {
+        match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => duration,
+        }
+    }
+
+    pub fn from_u32(duration: u32) -> Self {
+        if duration == 0 {
+            Self::Disabled
+        } else {
+            Self::Enabled { duration }
+        }
+    }
+}
+
+impl Default for Timer {
+    fn default() -> Self {
+        Self::Disabled
+    }
+}
+
+impl ToString for Timer {
+    fn to_string(&self) -> String {
+        self.to_u32().to_string()
+    }
+}
+
+impl FromStr for Timer {
+    type Err = ParseIntError;
+
+    fn from_str(input: &str) -> Result<Timer, ParseIntError> {
+        input.parse::<u32>().map(Self::from_u32)
+    }
+}
+
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
+    }
+}
+
+impl ChatId {
+    /// Get ephemeral message timer value in seconds.
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+        let timer = context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
+            )
+            .await?;
+        Ok(timer.unwrap_or_default())
+    }
+
+    /// Set ephemeral timer value without sending a message.
+    ///
+    /// Used when a message arrives indicating that someone else has
+    /// changed the timer value for a chat.
+    pub(crate) async fn inner_set_ephemeral_timer(
+        self,
+        context: &Context,
+        timer: Timer,
+    ) -> Result<(), Error> {
+        ensure!(!self.is_special(), "Invalid chat ID");
+
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+             SET ephemeral_timer=?
+             WHERE id=?;",
+                paramsv![timer, self],
+            )
+            .await?;
+
+        context.emit_event(EventType::ChatEphemeralTimerModified {
+            chat_id: self,
+            timer,
+        });
+        Ok(())
+    }
+
+    /// Set ephemeral message timer value in seconds.
+    ///
+    /// If timer value is 0, disable ephemeral message timer.
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+        if timer == self.get_ephemeral_timer(context).await? {
+            return Ok(());
+        }
+        self.inner_set_ephemeral_timer(context, timer).await?;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
+        if let Err(err) = send_msg(context, self, &mut msg).await {
+            error!(
+                context,
+                "Failed to send a message about ephemeral message timer change: {:?}", err
+            );
+        }
+        Ok(())
+    }
+}
+
+/// Returns a stock message saying that ephemeral timer is changed to `timer` by `from_id`.
+pub(crate) async fn stock_ephemeral_timer_changed(
+    context: &Context,
+    timer: Timer,
+    from_id: u32,
+) -> String {
+    let stock_message = match timer {
+        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+        Timer::Enabled { duration } => match duration {
+            60 => StockMessage::MsgEphemeralTimerMinute,
+            3600 => StockMessage::MsgEphemeralTimerHour,
+            86400 => StockMessage::MsgEphemeralTimerDay,
+            604_800 => StockMessage::MsgEphemeralTimerWeek,
+            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
+            _ => StockMessage::MsgEphemeralTimerEnabled,
+        },
+    };
+
+    context
+        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
+        .await
+}
+
+impl MsgId {
+    /// Returns ephemeral message timer value for the message.
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
+        let res = match context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
+            )
+            .await?
+        {
+            None | Some(0) => Timer::Disabled,
+            Some(duration) => Timer::Enabled { duration },
+        };
+        Ok(res)
+    }
+
+    /// Starts ephemeral message timer for the message if it is not started yet.
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
+        if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
+            let ephemeral_timestamp = time() + i64::from(duration);
+
+            context
+                .sql
+                .execute(
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
+                WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
+                AND id = ?",
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
+                )
+                .await?;
+            schedule_ephemeral_task(context).await;
+        }
+        Ok(())
+    }
+}
+
+/// Deletes messages which are expired according to
+/// `delete_device_after` setting or `ephemeral_timestamp` column.
+///
+/// Returns true if any message is deleted, so caller can emit
+/// MsgsChanged event. If nothing has been deleted, returns
+/// false. This function does not emit the MsgsChanged event itself,
+/// because it is also called when chatlist is reloaded, and emitting
+/// MsgsChanged there will cause infinite reload loop.
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+    let mut updated = context
+        .sql
+        .execute(
+            "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE \
+             ephemeral_timestamp != 0 \
+             AND ephemeral_timestamp < ? \
+             AND chat_id != ?",
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+        )
+        .await?
+        > 0;
+
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
+        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap_or_default()
+            .0;
+        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+            .await
+            .unwrap_or_default()
+            .0;
+
+        let threshold_timestamp = time() - delete_device_after;
+
+        // Delete expired messages
+        //
+        // Only update the rows that have to be updated, to avoid emitting
+        // unnecessary "chat modified" events.
+        let rows_modified = context
+            .sql
+            .execute(
+                "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE timestamp < ? \
+             AND chat_id > ? \
+             AND chat_id != ? \
+             AND chat_id != ?",
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
+            )
+            .await?;
+
+        updated |= rows_modified > 0;
+    }
+
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
+}
+
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
+///
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
+    let ephemeral_timestamp: Option<i64> = match context
+        .sql
+        .query_get_value_result(
+            "SELECT ephemeral_timestamp \
+         FROM msgs \
+         WHERE ephemeral_timestamp != 0 \
+           AND chat_id != ? \
+         ORDER BY ephemeral_timestamp ASC \
+         LIMIT 1",
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
+        )
+        .await
+    {
+        Err(err) => {
+            warn!(context, "Can't calculate next ephemeral timeout: {}", err);
+            return;
+        }
+        Ok(ephemeral_timestamp) => ephemeral_timestamp,
+    };
+
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
+
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+        let now = SystemTime::now();
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
+
+        if let Ok(duration) = until.duration_since(now) {
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                emit_event!(
+                    context1,
+                    EventType::MsgsChanged {
+                        chat_id: ChatId::new(0),
+                        msg_id: MsgId::new(0)
+                    }
+                );
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            emit_event!(
+                context,
+                EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0)
+                }
+            );
+        }
+    }
+}
+
+/// Returns ID of any expired message that should be deleted from the server.
+///
+/// It looks up the trash chat too, to find messages that are already
+/// deleted locally, but not deleted on the server.
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+    let now = time();
+
+    let threshold_timestamp = match context.get_config_delete_server_after().await {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
+
+    context
+        .sql
+        .query_row_optional(
+            "SELECT id FROM msgs \
+         WHERE ( \
+         timestamp < ? \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         ) \
+         AND server_uid != 0 \
+         LIMIT 1",
+            paramsv![threshold_timestamp, now],
+            |row| row.get::<_, MsgId>(0),
+        )
+        .await
+}
+
+/// Start ephemeral timers for seen messages if they are not started
+/// yet.
+///
+/// It is possible that timers are not started due to a missing or
+/// failed `MsgId.start_ephemeral_timer()` call, either in the current
+/// or previous version of Delta Chat.
+///
+/// This function is supposed to be called in the background,
+/// e.g. from housekeeping task.
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+    context
+        .sql
+        .execute(
+            "UPDATE msgs \
+    SET ephemeral_timestamp = ? + ephemeral_timer \
+    WHERE ephemeral_timer > 0 \
+    AND ephemeral_timestamp = 0 \
+    AND state NOT IN (?, ?, ?)",
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
+        )
+        .await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::*;
+
+    #[async_std::test]
+    async fn test_stock_ephemeral_messages() {
+        let context = TestContext::new().await.ctx;
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            "Message deletion timer is disabled by me."
+        );
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
+            "Message deletion timer is disabled."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
+            "Message deletion timer is set to 1 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
+            "Message deletion timer is set to 30 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
+            "Message deletion timer is set to 1 minute."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
+            "Message deletion timer is set to 1 hour."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 day."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 week."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 4 * 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 4 weeks."
+        );
+    }
+}

src/events.rs

@@ -1,10 +1,13 @@
 //! # Events specification
 
+use std::ops::Deref;
+
 use async_std::path::PathBuf;
 use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
 
 #[derive(Debug)]
@@ -53,14 +56,32 @@ impl EventEmitter {
         async_std::task::block_on(self.recv())
     }
 
-    /// Blocking async recv of an event. Return `None` if the `Sender` has been droped.
+    /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
         // TODO: change once we can use async channels internally.
         self.0.recv().await.ok()
     }
+
+    pub fn try_recv(&self) -> Result<Event, async_std::sync::TryRecvError> {
+        self.0.try_recv()
+    }
 }
 
-impl Event {
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Event {
+    pub id: u32,
+    pub typ: EventType,
+}
+
+impl Deref for Event {
+    type Target = EventType;
+
+    fn deref(&self) -> &EventType {
+        &self.typ
+    }
+}
+
+impl EventType {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
@@ -71,7 +92,7 @@ impl Event {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
-pub enum Event {
+pub enum EventType {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
@@ -106,7 +127,7 @@ pub enum Event {
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
-    /// Emitted when an new file in the $BLOBDIR was created
+    /// Emitted when an file in the $BLOBDIR was deleted
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -139,7 +160,6 @@ pub enum Event {
     /// Network errors should be reported to users in a non-disturbing way,
     /// however, as network errors may come in a sequence,
     /// it is not useful to raise each an every error to the user.
-    /// For this purpose, data1 is set to 1 if the error is probably worth reporting.
     ///
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
@@ -189,9 +209,19 @@ pub enum Event {
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// This event does not include ephemeral timer modification, which
+    /// is a separate event.
     #[strum(props(id = "2020"))]
     ChatModified(ChatId),
 
+    /// Chat ephemeral timer changed.
+    #[strum(props(id = "2021"))]
+    ChatEphemeralTimerModified {
+        chat_id: ChatId,
+        timer: EphemeralTimer,
+    },
+
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.

src/headerdef.rs

@@ -21,6 +21,7 @@ pub enum HeaderDef {
     References,
     InReplyTo,
     Precedence,
+    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -34,6 +35,7 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
+    ChatWebrtcRoom,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,
@@ -41,6 +43,7 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    EphemeralTimer,
     _TestHeader,
 }
 

src/imap/client.rs

@@ -56,7 +56,7 @@ impl Client {
     pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
         self,
         auth_type: S,
-        authenticator: &A,
+        authenticator: A,
     ) -> std::result::Result<Session, (ImapError, Self)> {
         let Client { inner, is_secure } = self;
         let session =
@@ -85,9 +85,6 @@ impl Client {
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
 
         let _greeting = client
             .read_response()
@@ -104,9 +101,6 @@ impl Client {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
         let _greeting = client
             .read_response()
             .await

src/imap/idle.rs

@@ -1,6 +1,7 @@
 use super::Imap;
 
 use async_imap::extensions::idle::IdleResponse;
+use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
@@ -13,19 +14,19 @@ type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
+    #[error("IMAP IDLE protocol failed to init/complete: {0}")]
     IdleProtocolFailed(#[from] async_imap::error::Error),
 
-    #[error("IMAP IDLE protocol timed out")]
+    #[error("IMAP IDLE protocol timed out: {0}")]
     IdleTimeout(#[from] async_std::future::TimeoutError),
 
     #[error("IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[error("IMAP select folder error")]
+    #[error("IMAP select folder error: {0}")]
     SelectFolderError(#[from] select_folder::Error),
 
-    #[error("Setup handle error")]
+    #[error("Setup handle error: {0}")]
     SetupHandleError(#[from] super::Error),
 }
 
@@ -48,11 +49,27 @@ impl Imap {
 
         self.select_folder(context, watch_folder.clone()).await?;
 
-        let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
-        if let Some(session) = session {
+        if let Some(session) = self.session.take() {
+            // if we have unsolicited responses we directly return
+            let mut unsolicited_exists = false;
+            while let Ok(response) = session.unsolicited_responses.try_recv() {
+                match response {
+                    UnsolicitedResponse::Exists(_) => {
+                        warn!(context, "skip idle, got unsolicited EXISTS {:?}", response);
+                        unsolicited_exists = true;
+                    }
+                    _ => info!(context, "ignoring unsolicited response {:?}", response),
+                }
+            }
+
+            if unsolicited_exists {
+                self.session = Some(session);
+                return Ok(info);
+            }
+
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
                 return Err(Error::IdleProtocolFailed(err));
@@ -65,68 +82,43 @@ impl Imap {
                 Interrupt(InterruptInfo),
             }
 
-            if self.skip_next_idle_wait {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait = false;
-                drop(idle_wait);
+            info!(context, "Idle entering wait-on-remote state");
+            let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
+                let probe_network = self.idle_interrupt.recv().await;
+
+                // cancel imap idle connection properly
                 drop(interrupt);
 
-                info!(context, "Idle wait was skipped");
-            } else {
-                info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
-                    self.idle_interrupt.recv().map(|probe_network| {
-                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
-                    }),
-                );
+                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+            });
 
-                match fut.await {
-                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
-                        info!(context, "Idle has NewData");
-                    }
-                    // TODO: idle_wait does not distinguish manual interrupts
-                    // from Timeouts if we would know it's a Timeout we could bail
-                    // directly and reconnect .
-                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                        info!(context, "Idle-wait timeout or interruption");
-                    }
-                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Ok(Event::Interrupt(i)) => {
-                        info = i;
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Err(err) => {
-                        warn!(context, "Idle wait errored: {:?}", err);
-                    }
+            match fut.await {
+                Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
+                    info!(context, "Idle has NewData {:?}", x);
+                }
+                Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                    info!(context, "Idle-wait timeout or interruption");
+                }
+                Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                    info!(context, "Idle wait was interrupted");
+                }
+                Ok(Event::Interrupt(i)) => {
+                    info = i;
+                    info!(context, "Idle wait was interrupted");
+                }
+                Err(err) => {
+                    warn!(context, "Idle wait errored: {:?}", err);
                 }
             }
 
-            // if we can't properly terminate the idle
-            // protocol let's break the connection.
-            let res = handle
+            let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(|err| {
-                    self.trigger_reconnect();
-                    Error::IdleTimeout(err)
-                })?;
-
-            match res {
-                Ok(session) => {
-                    self.session = Some(Session { inner: session });
-                }
-                Err(err) => {
-                    // if we cannot terminate IDLE it probably
-                    // means that we waited long (with idle_wait)
-                    // but the network went away/changed
-                    self.trigger_reconnect();
-                    return Err(Error::IdleProtocolFailed(err));
-                }
-            }
+                .map_err(Error::IdleTimeout)??;
+            self.session = Some(Session { inner: session });
+        } else {
+            warn!(context, "Attempted to idle without a session");
         }
 
         Ok(info)
@@ -148,73 +140,66 @@ impl Imap {
             return self.idle_interrupt.recv().await.unwrap_or_default();
         }
 
-        let mut info: InterruptInfo = Default::default();
-        if self.skip_next_idle_wait {
-            // interrupt_idle has happened before we
-            // provided self.interrupt
-            self.skip_next_idle_wait = false;
-            info!(context, "fake-idle wait was skipped");
-        } else {
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        // check every minute if there are new messages
+        // TODO: grow sleep durations / make them more flexible
+        let mut interval = async_std::stream::interval(Duration::from_secs(60));
 
-            enum Event {
-                Tick,
-                Interrupt(InterruptInfo),
-            }
-            // loop until we are interrupted or if we fetched something
-            info =
-                loop {
-                    use futures::future::FutureExt;
-                    match interval
-                        .next()
-                        .map(|_| Event::Tick)
-                        .race(self.idle_interrupt.recv().map(|probe_network| {
-                            Event::Interrupt(probe_network.unwrap_or_default())
-                        }))
-                        .await
-                    {
-                        Event::Tick => {
-                            // try to connect with proper login params
-                            // (setup_handle_if_needed might not know about them if we
-                            // never successfully connected)
-                            if let Err(err) = self.connect_configured(context).await {
-                                warn!(context, "fake_idle: could not connect: {}", err);
-                                continue;
-                            }
-                            if self.config.can_idle {
-                                // we only fake-idled because network was gone during IDLE, probably
-                                break InterruptInfo::new(false, None);
-                            }
-                            info!(context, "fake_idle is connected");
-                            // we are connected, let's see if fetching messages results
-                            // in anything.  If so, we behave as if IDLE had data but
-                            // will have already fetched the messages so perform_*_fetch
-                            // will not find any new.
+        enum Event {
+            Tick,
+            Interrupt(InterruptInfo),
+        }
+        // loop until we are interrupted or if we fetched something
+        let info = loop {
+            use futures::future::FutureExt;
+            match interval
+                .next()
+                .map(|_| Event::Tick)
+                .race(
+                    self.idle_interrupt
+                        .recv()
+                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                )
+                .await
+            {
+                Event::Tick => {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context).await {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break InterruptInfo::new(false, None);
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
 
-                            if let Some(ref watch_folder) = watch_folder {
-                                match self.fetch_new_messages(context, watch_folder).await {
-                                    Ok(res) => {
-                                        info!(context, "fetch_new_messages returned {:?}", res);
-                                        if res {
-                                            break InterruptInfo::new(false, None);
-                                        }
-                                    }
-                                    Err(err) => {
-                                        error!(context, "could not fetch from folder: {}", err);
-                                        self.trigger_reconnect()
-                                    }
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break InterruptInfo::new(false, None);
                                 }
                             }
-                        }
-                        Event::Interrupt(info) => {
-                            // Interrupt
-                            break info;
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
                         }
                     }
-                };
-        }
+                }
+                Event::Interrupt(info) => {
+                    // Interrupt
+                    break info;
+                }
+            }
+        };
 
         info!(
             context,

src/imap/mod.rs

@@ -19,16 +19,16 @@ use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf, from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list,
 };
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
-use crate::message::{self, update_server_uid};
+use crate::message::{self, update_server_uid, MessageState};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::provider::get_provider_info;
-use crate::{scheduler::InterruptInfo, stock::StockMessage};
+use crate::{chat, scheduler::InterruptInfo, stock::StockMessage};
 
 mod client;
 mod idle;
@@ -36,6 +36,7 @@ pub mod select_folder;
 mod session;
 
 use client::Client;
+use message::Message;
 use session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
@@ -115,8 +116,8 @@ pub struct Imap {
     session: Option<Session>,
     connected: bool,
     interrupt: Option<stop_token::StopSource>,
-    skip_next_idle_wait: bool,
     should_reconnect: bool,
+    login_failed_once: bool,
 }
 
 #[derive(Debug)]
@@ -128,7 +129,7 @@ struct OAuth2 {
 impl async_imap::Authenticator for OAuth2 {
     type Response = String;
 
-    fn process(&self, _data: &[u8]) -> Self::Response {
+    fn process(&mut self, _data: &[u8]) -> Self::Response {
         format!(
             "user={}\x01auth=Bearer {}\x01\x01",
             self.user, self.access_token
@@ -189,8 +190,8 @@ impl Imap {
             session: Default::default(),
             connected: Default::default(),
             interrupt: Default::default(),
-            skip_next_idle_wait: Default::default(),
             should_reconnect: Default::default(),
+            login_failed_once: Default::default(),
         }
     }
 
@@ -261,7 +262,7 @@ impl Imap {
                             user: imap_user.into(),
                             access_token: token,
                         };
-                        client.authenticate("XOAUTH2", &auth).await
+                        client.authenticate("XOAUTH2", auth).await
                     } else {
                         return Err(Error::OauthError);
                     }
@@ -283,7 +284,7 @@ impl Imap {
                         .await
                 };
                 // IMAP connection failures are reported to users
-                emit_event!(context, Event::ErrorNetwork(message));
+                emit_event!(context, EventType::ErrorNetwork(message));
                 return Err(Error::ConnectionFailed(err.to_string()));
             }
         };
@@ -295,18 +296,40 @@ impl Imap {
                 // needs to be set here to ensure it is set on reconnects.
                 self.connected = true;
                 self.session = Some(session);
+                self.login_failed_once = false;
                 Ok(())
             }
+
             Err((err, _)) => {
                 let imap_user = self.config.imap_user.to_owned();
                 let message = context
                     .stock_string_repl_str(StockMessage::CannotLogin, &imap_user)
                     .await;
 
-                emit_event!(
-                    context,
-                    Event::ErrorNetwork(format!("{} ({})", message, err))
-                );
+                warn!(context, "{} ({})", message, err);
+                emit_event!(context, EventType::ErrorNetwork(message.clone()));
+
+                let lock = context.wrong_pw_warning_mutex.lock().await;
+                if self.login_failed_once
+                    && context.get_config_bool(Config::NotifyAboutWrongPw).await
+                {
+                    if let Err(e) = context.set_config(Config::NotifyAboutWrongPw, None).await {
+                        warn!(context, "{}", e);
+                    }
+                    drop(lock);
+
+                    let mut msg = Message::new(Viewtype::Text);
+                    msg.text = Some(message);
+                    if let Err(e) =
+                        chat::add_device_msg_with_importance(context, None, Some(&mut msg), true)
+                            .await
+                    {
+                        warn!(context, "{}", e);
+                    }
+                } else {
+                    self.login_failed_once = true;
+                }
+
                 self.trigger_reconnect();
                 Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
             }
@@ -422,7 +445,7 @@ impl Imap {
                         self.connected = true;
                         emit_event!(
                             context,
-                            Event::ImapConnected(format!(
+                            EventType::ImapConnected(format!(
                                 "IMAP-LOGIN as {}, capabilities: {}",
                                 lp.mail_user, caps_list,
                             ))
@@ -550,8 +573,14 @@ impl Imap {
                     let set = format!("{}", mailbox.exists);
                     match session.fetch(set, JUST_UID).await {
                         Ok(mut list) => {
-                            if let Some(Ok(msg)) = list.next().await {
-                                msg.uid.unwrap_or_default()
+                            let mut new_last_seen_uid = None;
+                            while let Some(fetch) = list.next().await.transpose()? {
+                                if fetch.message == mailbox.exists && fetch.uid.is_some() {
+                                    new_last_seen_uid = fetch.uid;
+                                }
+                            }
+                            if let Some(new_last_seen_uid) = new_last_seen_uid {
+                                new_last_seen_uid
                             } else {
                                 return Err(Error::Other("failed to fetch".into()));
                             }
@@ -732,9 +761,17 @@ impl Imap {
         folder: S,
         server_uids: &[u32],
     ) -> (Option<u32>, usize) {
-        if server_uids.is_empty() {
-            return (None, 0);
-        }
+        let set = match server_uids {
+            [] => return (None, 0),
+            [server_uid] => server_uid.to_string(),
+            [first_uid, .., last_uid] => {
+                // XXX: it is assumed that UIDs are sorted and
+                // contiguous. If UIDs are not contiguous, more
+                // messages than needed will be downloaded.
+                debug_assert!(first_uid < last_uid, "uids must be sorted");
+                format!("{}:{}", first_uid, last_uid)
+            }
+        };
 
         if !self.is_connected() {
             warn!(context, "Not connected");
@@ -750,15 +787,6 @@ impl Imap {
 
         let session = self.session.as_mut().unwrap();
 
-        let set = if server_uids.len() == 1 {
-            server_uids[0].to_string()
-        } else {
-            let first_uid = server_uids[0];
-            let last_uid = server_uids[server_uids.len() - 1];
-            assert!(first_uid < last_uid, "uids must be sorted");
-            format!("{}:{}", first_uid, last_uid)
-        };
-
         let mut msgs = match session.uid_fetch(&set, BODY_FLAGS).await {
             Ok(msgs) => msgs,
             Err(err) => {
@@ -782,7 +810,6 @@ impl Imap {
         let mut last_uid = None;
         let mut count = 0;
 
-        let mut tasks = Vec::with_capacity(server_uids.len());
         while let Some(Ok(msg)) = msgs.next().await {
             let server_uid = msg.uid.unwrap_or_default();
 
@@ -802,32 +829,17 @@ impl Imap {
             let context = context.clone();
             let folder = folder.clone();
 
-            let task = async_std::task::spawn(async move {
-                // safe, as we checked above that there is a body.
-                let body = msg.body().unwrap();
-                let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
+            // safe, as we checked above that there is a body.
+            let body = msg.body().unwrap();
+            let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
-                match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
-                    Ok(_) => Some(server_uid),
-                    Err(err) => {
-                        warn!(context, "dc_receive_imf error: {}", err);
-                        read_errors += 1;
-                        None
-                    }
-                }
-            });
-            tasks.push(task);
-        }
-
-        for task in futures::future::join_all(tasks).await {
-            match task {
-                Some(uid) => {
-                    last_uid = Some(uid);
-                }
-                None => {
+            match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
+                Ok(_) => last_uid = Some(server_uid),
+                Err(err) => {
+                    warn!(context, "dc_receive_imf error: {}", err);
                     read_errors += 1;
                 }
-            }
+            };
         }
 
         if count != server_uids.len() {
@@ -876,7 +888,7 @@ impl Imap {
                     Ok(_) => {
                         emit_event!(
                             context,
-                            Event::ImapMessageMoved(format!(
+                            EventType::ImapMessageMoved(format!(
                                 "IMAP Message {} moved to {}",
                                 display_folder_id, dest_folder
                             ))
@@ -920,7 +932,7 @@ impl Imap {
             warn!(context, "Cannot mark {} as \"Deleted\" after copy.", uid);
             emit_event!(
                 context,
-                Event::ImapMessageMoved(format!(
+                EventType::ImapMessageMoved(format!(
                     "IMAP Message {} copied to {} (delete FAILED)",
                     display_folder_id, dest_folder
                 ))
@@ -930,7 +942,7 @@ impl Imap {
             self.config.selected_folder_needs_expunge = true;
             emit_event!(
                 context,
-                Event::ImapMessageMoved(format!(
+                EventType::ImapMessageMoved(format!(
                     "IMAP Message {} copied to {} (delete successfull)",
                     display_folder_id, dest_folder
                 ))
@@ -964,7 +976,11 @@ impl Imap {
         if let Some(ref mut session) = &mut self.session {
             let query = format!("+FLAGS ({})", flag);
             match session.uid_store(uid_set, &query).await {
-                Ok(_) => {}
+                Ok(mut responses) => {
+                    while let Some(_response) = responses.next().await {
+                        // Read all the responses
+                    }
+                }
                 Err(err) => {
                     warn!(
                         context,
@@ -1067,8 +1083,35 @@ impl Imap {
         if let Some(ref mut session) = &mut self.session {
             match session.uid_fetch(set, DELETE_CHECK_FLAGS).await {
                 Ok(mut msgs) => {
-                    let fetch = if let Some(Ok(fetch)) = msgs.next().await {
-                        fetch
+                    let mut remote_message_id = None;
+
+                    while let Some(response) = msgs.next().await {
+                        match response {
+                            Ok(fetch) => {
+                                if fetch.uid == Some(uid) {
+                                    remote_message_id = get_fetch_headers(&fetch)
+                                        .and_then(|headers| prefetch_get_message_id(&headers))
+                                        .ok();
+                                }
+                            }
+                            Err(err) => {
+                                warn!(context, "IMAP fetch error {}", err);
+                                return ImapActionResult::RetryLater;
+                            }
+                        }
+                    }
+
+                    if let Some(remote_message_id) = remote_message_id {
+                        if remote_message_id != message_id {
+                            warn!(
+                                context,
+                                "Cannot delete on IMAP, {}: remote message-id '{}' != '{}'",
+                                display_imap_id,
+                                remote_message_id,
+                                message_id,
+                            );
+                            return ImapActionResult::Failed;
+                        }
                     } else {
                         warn!(
                             context,
@@ -1077,21 +1120,6 @@ impl Imap {
                             message_id,
                         );
                         return ImapActionResult::AlreadyDone;
-                    };
-
-                    let remote_message_id = get_fetch_headers(&fetch)
-                        .and_then(|headers| prefetch_get_message_id(&headers))
-                        .unwrap_or_default();
-
-                    if remote_message_id != message_id {
-                        warn!(
-                            context,
-                            "Cannot delete on IMAP, {}: remote message-id '{}' != '{}'",
-                            display_imap_id,
-                            remote_message_id,
-                            message_id,
-                        );
-                        return ImapActionResult::Failed;
                     }
                 }
                 Err(err) => {
@@ -1114,7 +1142,7 @@ impl Imap {
         } else {
             emit_event!(
                 context,
-                Event::ImapMessageDeleted(format!(
+                EventType::ImapMessageDeleted(format!(
                     "IMAP Message {} marked as deleted [{}]",
                     display_imap_id, message_id
                 ))
@@ -1288,7 +1316,7 @@ impl Imap {
         self.config.selected_folder_needs_expunge = true;
         match self.select_folder::<String>(context, None).await {
             Ok(()) => {
-                emit_event!(context, Event::ImapFolderEmptied(folder.to_string()));
+                emit_event!(context, EventType::ImapFolderEmptied(folder.to_string()));
             }
             Err(err) => {
                 error!(context, "expunge failed {}: {:?}", folder, err);
@@ -1360,14 +1388,26 @@ async fn precheck_imf(
             let delete_server_after = context.get_config_delete_server_after().await;
 
             if delete_server_after != Some(0) {
-                context
-                    .do_heuristics_moves(server_folder.as_ref(), msg_id)
+                if msg_id
+                    .needs_move(context, server_folder)
+                    .await
+                    .unwrap_or_default()
+                {
+                    // If the bcc-self message is not moved, directly
+                    // add MarkSeen job, otherwise MarkSeen job is
+                    // added after the Move Job completed.
+                    job::add(
+                        context,
+                        job::Job::new(Action::MoveMsg, msg_id.to_u32(), Params::new(), 0),
+                    )
                     .await;
-                job::add(
-                    context,
-                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
-                )
-                .await;
+                } else {
+                    job::add(
+                        context,
+                        job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                    )
+                    .await;
+                }
             }
         } else if old_server_folder != server_folder {
             info!(
@@ -1402,6 +1442,13 @@ async fn precheck_imf(
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
             update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
+            if let Ok(MessageState::InSeen) = msg_id.get_state(context).await {
+                job::add(
+                    context,
+                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                )
+                .await;
+            };
             context
                 .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
                 .await;
@@ -1449,7 +1496,7 @@ async fn prefetch_is_reply_to_chat_message(
     false
 }
 
-async fn prefetch_should_download(
+pub(crate) async fn prefetch_should_download(
     context: &Context,
     headers: &[mailparse::MailHeader<'_>],
     show_emails: ShowEmails,
@@ -1457,6 +1504,13 @@ async fn prefetch_should_download(
     let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
     let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await;
 
+    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
+        let from = from.to_ascii_lowercase();
+        from.contains("mailer-daemon") || from.contains("mail-daemon")
+    } else {
+        false
+    };
+
     // Autocrypt Setup Message should be shown even if it is from non-chat client.
     let is_autocrypt_setup_message = headers
         .get_header_value(HeaderDef::AutocryptSetupMessage)
@@ -1474,8 +1528,8 @@ async fn prefetch_should_download(
             }
             ShowEmails::All => true,
         };
-    let show = show && !blocked_contact;
-    Ok(show)
+    let should_download = (show && !blocked_contact) || maybe_ndn;
+    Ok(should_download)
 }
 
 async fn message_needs_processing(

src/imap/select_folder.rs

@@ -51,6 +51,14 @@ impl Imap {
         Ok(())
     }
 
+    /// Issues a CLOSE command if selected folder needs expunge.
+    pub(crate) async fn maybe_close_folder(&mut self, context: &Context) -> Result<()> {
+        if self.config.selected_folder_needs_expunge {
+            self.close_folder(context).await?;
+        }
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -76,10 +84,7 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.selected_folder_needs_expunge };
-        if needs_expunge {
-            self.close_folder(context).await?;
-        }
+        self.maybe_close_folder(context).await?;
 
         // select new folder
         if let Some(ref folder) = folder {

src/imex.rs

@@ -1,10 +1,16 @@
 //! # Import/export module
 
 use std::any::Any;
-use std::cmp::{max, min};
+use std::{
+    cmp::{max, min},
+    ffi::OsStr,
+};
 
 use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
+use async_std::{
+    fs::{self, File},
+    prelude::*,
+};
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -16,7 +22,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -24,6 +30,11 @@ use crate::param::*;
 use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock::StockMessage;
+use async_tar::Archive;
+
+// Name of the database file in the backup.
+const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
+const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 #[repr(i32)]
@@ -42,8 +53,8 @@ pub enum ImexMode {
     /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
-    /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
-    /// the format is `delta-chat.<day>-<number>.bak`
+    /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
+    /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
     /// `param1` is the file (not: directory) to import. The file is normally
@@ -85,6 +96,37 @@ pub async fn imex(
 
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+    let dir_name = dir_name.as_ref();
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut newest_backup_name = "".to_string();
+    let mut newest_backup_path: Option<PathBuf> = None;
+
+    while let Some(dirent) = dir_iter.next().await {
+        if let Ok(dirent) = dirent {
+            let path = dirent.path();
+            let name = dirent.file_name();
+            let name: String = name.to_string_lossy().into();
+            if name.starts_with("delta-chat")
+                && name.ends_with(".tar")
+                && (newest_backup_name.is_empty() || name > newest_backup_name)
+            {
+                // We just use string comparison to determine which backup is newer.
+                // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
+                newest_backup_path = Some(path);
+                newest_backup_name = name;
+            }
+        }
+    }
+
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => has_backup_old(context, dir_name).await,
+        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
+    }
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
@@ -178,10 +220,11 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 ///
 /// The `passphrase` must be at least 2 characters long.
 pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
-    ensure!(
-        passphrase.len() >= 2,
-        "Passphrase must be at least 2 chars long."
-    );
+    let passphrase_begin = if let Some(passphrase_begin) = passphrase.get(..2) {
+        passphrase_begin
+    } else {
+        bail!("Passphrase must be at least 2 chars long.");
+    };
     let private_key = SignedSecretKey::load_self(context).await?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
@@ -196,7 +239,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
             "Passphrase-Format: numeric9x4\r\n",
             "Passphrase-Begin: {}"
         ),
-        &passphrase[..2]
+        passphrase_begin
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
@@ -368,11 +411,11 @@ async fn imex_inner(
     ensure!(param.is_some(), "No Import/export dir/file given.");
 
     info!(context, "Import/export process started.");
-    context.emit_event(Event::ImexProgress(10));
+    context.emit_event(EventType::ImexProgress(10));
 
     ensure!(context.sql.is_open().await, "Database not opened.");
 
-    let path = param.unwrap();
+    let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?;
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -385,18 +428,22 @@ async fn imex_inner(
     let success = match what {
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
-        ImexMode::ExportBackup => export_backup(context, path).await,
+
+        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
+        // (now is 07/2020)
+        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
     };
 
     match success {
         Ok(()) => {
             info!(context, "IMEX successfully completed");
-            context.emit_event(Event::ImexProgress(1000));
+            context.emit_event(EventType::ImexProgress(1000));
             Ok(())
         }
         Err(err) => {
-            context.emit_event(Event::ImexProgress(0));
+            context.emit_event(EventType::ImexProgress(0));
             bail!("IMEX FAILED to complete: {}", err);
         }
     }
@@ -404,6 +451,75 @@ async fn imex_inner(
 
 /// Import Backup
 async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+    if backup_to_import
+        .as_ref()
+        .to_string_lossy()
+        .ends_with(".bak")
+    {
+        // Backwards compability
+        return import_backup_old(context, backup_to_import).await;
+    }
+
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.as_ref().display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !context.is_configured().await,
+        "Cannot import backups to accounts in use."
+    );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
+
+    let backup_file = File::open(backup_to_import).await?;
+    let archive = Archive::new(backup_file);
+    let mut entries = archive.entries()?;
+    while let Some(file) = entries.next().await {
+        let f = &mut file?;
+        if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
+            // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
+            f.unpack_in(context.get_blobdir()).await?;
+            fs::rename(
+                context.get_blobdir().join(DBFILE_BACKUP_NAME),
+                context.get_dbfile(),
+            )
+            .await?;
+            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
+        } else {
+            // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
+            f.unpack_in(context.get_blobdir()).await?;
+            let from_path = context.get_blobdir().join(f.path()?);
+            if from_path.is_file().await {
+                if let Some(name) = from_path.file_name() {
+                    fs::rename(&from_path, context.get_blobdir().join(name)).await?;
+                } else {
+                    warn!(context, "No file name");
+                }
+            }
+        }
+    }
+
+    ensure!(
+        context
+            .sql
+            .open(&context, &context.get_dbfile(), false)
+            .await,
+        "could not re-open db"
+    );
+
+    delete_and_reset_all_device_msgs(&context).await?;
+
+    Ok(())
+}
+
+async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -448,27 +564,33 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
     );
 
-    let files = context
+    // Load IDs only for now, without the file contents, to avoid
+    // consuming too much memory.
+    let file_ids = context
         .sql
         .query_map(
-            "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
+            "SELECT id FROM backup_blobs ORDER BY id",
             paramsv![],
-            |row| {
-                let name: String = row.get(0)?;
-                let blob: Vec<u8> = row.get(1)?;
-
-                Ok((name, blob))
-            },
-            |files| {
-                files
-                    .collect::<std::result::Result<Vec<_>, _>>()
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
                     .map_err(Into::into)
             },
         )
         .await?;
 
     let mut all_files_extracted = true;
-    for (processed_files_cnt, (file_name, file_blob)) in files.into_iter().enumerate() {
+    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        // Load a single blob into memory
+        let (file_name, file_blob) = context
+            .sql
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?)),
+            )
+            .await?;
+
         if context.shall_stop_ongoing().await {
             all_files_extracted = false;
             break;
@@ -480,7 +602,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         if permille > 990 {
             permille = 990
         }
-        context.emit_event(Event::ImexProgress(permille));
+        context.emit_event(EventType::ImexProgress(permille));
         if file_blob.is_empty() {
             continue;
         }
@@ -505,14 +627,83 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
-The macro avoids weird values of 0% or 100% while still working. */
+#[allow(unused)]
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
+    let now = time();
+    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+
+    context
+        .sql
+        .set_raw_config_int(context, "backup_time", now as i32)
+        .await?;
+    sql::housekeeping(context).await;
+
+    context
+        .sql
+        .execute("VACUUM;", paramsv![])
+        .await
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
+
+    // we close the database during the export
+    context.sql.close().await;
+
+    info!(
+        context,
+        "Backup '{}' to '{}'.",
+        context.get_dbfile().display(),
+        dest_path.display(),
+    );
+
+    let res = export_backup_inner(context, &temp_path).await;
+
+    // we re-open the database after export is finished
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await;
+
+    match &res {
+        Ok(_) => {
+            fs::rename(temp_path, &dest_path).await?;
+            context.emit_event(EventType::ImexFileWritten(dest_path));
+        }
+        Err(e) => {
+            error!(context, "backup failed: {}", e);
+            // Not using dc_delete_file() here because it would send a DeletedBlobFile event
+            fs::remove_file(temp_path).await?;
+        }
+    }
+
+    res
+}
+
+async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
+    let file = File::create(temp_path).await?;
+
+    let mut builder = async_tar::Builder::new(file);
+
+    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
+    builder
+        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
+        .await?;
+
+    context.emit_event(EventType::ImexProgress(500));
+
+    builder
+        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
+        .await?;
+
+    builder.finish().await?;
+    Ok(())
+}
+
+async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
+    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
     let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
     sql::housekeeping(context).await;
@@ -556,7 +747,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
             dest_sql
                 .set_raw_config_int(context, "backup_time", now as i32)
                 .await?;
-            context.emit_event(Event::ImexFileWritten(dest_path_filename));
+            context.emit_event(EventType::ImexFileWritten(dest_path_filename));
             Ok(())
         }
     };
@@ -595,7 +786,7 @@ async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
             }
             processed_files_cnt += 1;
             let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-            context.emit_event(Event::ImexProgress(permille));
+            context.emit_event(EventType::ImexProgress(permille));
 
             let name_f = entry.file_name();
             let name = name_f.to_string_lossy();
@@ -762,7 +953,7 @@ where
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
-        context.emit_event(Event::ImexFileWritten(file_name));
+        context.emit_event(EventType::ImexFileWritten(file_name));
     }
     res
 }
@@ -776,9 +967,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
@@ -795,12 +986,12 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
@@ -808,7 +999,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_setup_code() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let setupcode = create_setup_code(&t.ctx);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
@@ -823,7 +1014,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_export_key_to_asc_file() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)

src/job.rs

@@ -21,8 +21,9 @@ use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::load_imap_deletion_msgid;
 use crate::error::{bail, ensure, format_err, Error, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::imap::*;
 use crate::location;
 use crate::login_param::LoginParam;
@@ -94,7 +95,6 @@ pub enum Action {
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
     EmptyServer = 107,
-    OldDeleteMsgOnImap = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -123,7 +123,6 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
-            OldDeleteMsgOnImap => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
             EmptyServer => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
@@ -254,40 +253,48 @@ impl Job {
 
                 let res = match err {
                     async_smtp::smtp::error::Error::Permanent(ref response) => {
-                        match response.code {
+                        // Workaround for incorrectly configured servers returning permanent errors
+                        // instead of temporary ones.
+                        let maybe_transient = match response.code {
                             // Sometimes servers send a permanent error when actually it is a temporary error
                             // For documentation see https://tools.ietf.org/html/rfc3463
-
-                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
                             Code {
                                 category: Category::MailSystem,
                                 detail: Detail::Zero,
                                 ..
-                            } => Status::RetryLater,
-
-                            _ => {
-                                // If we do not retry, add an info message to the chat
-                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
-                                match Message::load_from_db(context, MsgId::new(self.foreign_id))
-                                    .await
-                                {
-                                    Ok(message) => {
-                                        chat::add_info_msg(
-                                            context,
-                                            message.chat_id,
-                                            err.to_string(),
-                                        )
-                                        .await
-                                    }
-                                    Err(e) => warn!(
-                                        context,
-                                        "couldn't load chat_id to inform user about SMTP error: {}",
-                                        e
-                                    ),
-                                };
-
-                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } => {
+                                // Ignore status code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                                // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
+                                // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
+                                //
+                                // Other enhanced status codes, such as Postfix
+                                // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
+                                // are not ignored.
+                                response.message.get(0) == Some(&"5.5.0".to_string())
                             }
+                            _ => false,
+                        };
+
+                        if maybe_transient {
+                            Status::RetryLater
+                        } else {
+                            // If we do not retry, add an info message to the chat.
+                            // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
+                            // should definitely go here, because user has to open the link to
+                            // resume message sending.
+                            let msg_id = MsgId::new(self.foreign_id);
+                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+                            match Message::load_from_db(context, msg_id).await {
+                                Ok(message) => {
+                                    chat::add_info_msg(context, message.chat_id, err.to_string())
+                                        .await
+                                }
+                                Err(e) => error!(
+                                    context,
+                                    "couldn't load chat_id to inform user about SMTP error: {}", e
+                                ),
+                            };
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
                     async_smtp::smtp::error::Error::Transient(_) => {
@@ -624,7 +631,9 @@ impl Job {
             }
         }
         if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            imap.empty_folder(context, "INBOX").await;
+            if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
+                imap.empty_folder(context, &inbox_folder).await;
+            }
         }
         Status::Finished(Ok(()))
     }
@@ -638,7 +647,21 @@ impl Job {
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
-        match imap.set_seen(context, folder, msg.server_uid).await {
+
+        let result = if msg.server_uid == 0 {
+            // The message is moved or deleted by us.
+            //
+            // Do not call set_seen with zero UID, as it will return
+            // ImapActionResult::RetryLater, but we do not want to
+            // retry. If the message was moved, we will create another
+            // job to mark the message as seen later. If it was
+            // deleted, there is nothing to do.
+            ImapActionResult::Failed
+        } else {
+            imap.set_seen(context, folder, msg.server_uid).await
+        };
+
+        match result {
             ImapActionResult::RetryLater => Status::RetryLater,
             ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
             ImapActionResult::Success | ImapActionResult::Failed => {
@@ -646,7 +669,18 @@ impl Job {
                 // we want to send out an MDN anyway
                 // The job will not be retried so locally
                 // there is no risk of double-sending MDNs.
+                //
+                // Read receipts for system messages are never
+                // sent. These messages have no place to display
+                // received read receipt anyway.  And since their text
+                // is locally generated, quoting them is dangerous as
+                // it may contain contact names. E.g., for original
+                // message "Group left by me", a read receipt will
+                // quote "Group left by <name>", and the name can be a
+                // display name stored in address book rather than
+                // the name sent in the From field by the user.
                 if msg.param.get_bool(Param::WantsMdn).unwrap_or_default()
+                    && !msg.is_system_message()
                     && context.get_config_bool(Config::MdnsEnabled).await
                 {
                     if let Err(err) = send_mdn(context, &msg).await {
@@ -703,7 +737,7 @@ async fn set_delivered(context: &Context, msg_id: MsgId) {
         )
         .await
         .unwrap_or_default();
-    context.emit_event(Event::MsgDelivered { chat_id, msg_id });
+    context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
 }
 
 /// Constructs a job for sending a message.
@@ -806,7 +840,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
 
     if rendered_msg.is_encrypted && !needs_encryption {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
-        msg.save_param_to_disk(context).await;
+        msg.update_param(context).await;
     }
 
     ensure!(!recipients.is_empty(), "no recipients for smtp job set");
@@ -828,25 +862,6 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
-    if let Some(delete_server_after) = context.get_config_delete_server_after().await {
-        let threshold_timestamp = time() - delete_server_after;
-
-        context
-            .sql
-            .query_row_optional(
-                "SELECT id FROM msgs \
-             WHERE timestamp < ? \
-             AND server_uid != 0",
-                paramsv![threshold_timestamp],
-                |row| row.get::<_, MsgId>(0),
-            )
-            .await
-    } else {
-        Ok(None)
-    }
-}
-
 async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         Some(Job::new(
@@ -970,7 +985,6 @@ async fn perform_job_action(
             location::job_maybe_send_locations_ended(context, job).await
         }
         Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
-        Action::OldDeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
@@ -980,10 +994,7 @@ async fn perform_job_action(
         }
     };
 
-    info!(
-        context,
-        "Inbox finished immediate try {} of job {}", tries, job
-    );
+    info!(context, "Finished immediate try {} of job {}", tries, job);
 
     try_res
 }
@@ -1031,7 +1042,6 @@ pub async fn add(context: &Context, job: Job) {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
             | Action::EmptyServer
-            | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
             | Action::MarkseenMsgOnImap
             | Action::MoveMsg => {
@@ -1209,7 +1219,7 @@ mod tests {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
         let jobs = load_next(
             &t.ctx,
@@ -1231,7 +1241,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_load_next_job_one() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         insert_job(&t.ctx, 1).await;
 

src/key.rs

@@ -247,7 +247,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
         Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
+            let start = std::time::SystemTime::now();
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
@@ -258,7 +258,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
+                start.elapsed().unwrap_or_default().as_secs()
             );
             Ok(keypair)
         }
@@ -550,8 +550,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = dummy_context().await;
-        configure_alice_keypair(&t.ctx).await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
         assert_eq!(alice.public, pubkey);
         let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
@@ -560,7 +560,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 
     #[async_std::test]
     async fn test_load_self_generate_public() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -571,7 +571,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 
     #[async_std::test]
     async fn test_load_self_generate_secret() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -584,7 +584,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -611,7 +611,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let ctx = Arc::new(t.ctx);
 
         let ctx1 = ctx.clone();

src/keyring.rs

@@ -79,8 +79,8 @@ mod tests {
     #[async_std::test]
     async fn test_keyring_load_self() {
         // new_self() implies load_self()
-        let t = dummy_context().await;
-        configure_alice_keypair(&t.ctx).await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let alice = alice_keypair();
 
         let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();

src/lib.rs

@@ -1,6 +1,11 @@
 #![forbid(unsafe_code)]
-#![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![allow(clippy::match_bool)]
+#![deny(
+    clippy::correctness,
+    missing_debug_implementations,
+    clippy::all,
+    clippy::indexing_slicing
+)]
+#![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
 #[macro_use]
 extern crate num_derive;
@@ -43,6 +48,7 @@ pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
+pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;
@@ -73,6 +79,8 @@ mod dehtml;
 pub mod dc_receive_imf;
 pub mod dc_tools;
 
+pub mod accounts;
+
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed
 pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 

src/location.rs

@@ -9,7 +9,7 @@ use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -227,7 +227,7 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
                     .await;
                 chat::add_info_msg(context, chat_id, stock_str).await;
             }
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
             if 0 != seconds {
                 schedule_maybe_send_locations(context, false).await;
                 job::add(
@@ -301,7 +301,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             }
         }
         if continue_streaming {
-            context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
         schedule_maybe_send_locations(context, false).await;
     }
@@ -381,7 +381,7 @@ pub async fn delete_all(context: &Context) -> Result<(), Error> {
         .sql
         .execute("DELETE FROM locations;", paramsv![])
         .await?;
-    context.emit_event(Event::LocationChanged(None));
+    context.emit_event(EventType::LocationChanged(None));
     Ok(())
 }
 
@@ -530,7 +530,7 @@ pub async fn save(
             accuracy,
             ..
         } = location;
-        context
+        let (loc_id, ts) = context
             .sql
             .with_conn(move |mut conn| {
                 let mut stmt_test = conn
@@ -569,9 +569,11 @@ pub async fn save(
                         )?;
                     }
                 }
-                Ok(())
+                Ok((newest_location_id, newest_timestamp))
             })
             .await?;
+        newest_timestamp = ts;
+        newest_location_id = loc_id;
     }
 
     Ok(newest_location_id)
@@ -713,7 +715,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
                 .await;
             chat::add_info_msg(context, chat_id, stock_str).await;
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
         }
     }
     job::Status::Finished(Ok(()))
@@ -722,11 +724,11 @@ pub(crate) async fn job_maybe_send_locations_ended(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_kml_parse() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
 
         let xml =
             b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";

src/log.rs

@@ -11,7 +11,7 @@ macro_rules! info {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Info(full));
+        emit_event!($ctx, $crate::EventType::Info(full));
     }};
 }
 
@@ -26,7 +26,7 @@ macro_rules! warn {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Warning(full));
+        emit_event!($ctx, $crate::EventType::Warning(full));
     }};
 }
 
@@ -37,7 +37,18 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::Event::Error(formatted));
+        emit_event!($ctx, $crate::EventType::Error(formatted));
+    }};
+}
+
+#[macro_export]
+macro_rules! error_network {
+    ($ctx:expr, $msg:expr) => {
+        error_network!($ctx, $msg,)
+    };
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+        let formatted = format!($msg, $($args),*);
+        emit_event!($ctx, $crate::EventType::ErrorNetwork(formatted));
     }};
 }
 

src/login_param.rs

@@ -5,7 +5,7 @@ use std::fmt;
 
 use crate::context::Context;
 
-#[derive(Copy, Clone, Debug, Display, FromPrimitive)]
+#[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(i32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
@@ -25,7 +25,7 @@ impl Default for CertificateChecks {
     }
 }
 
-#[derive(Default, Debug)]
+#[derive(Default, Debug, Clone)]
 pub struct LoginParam {
     pub addr: String,
     pub mail_server: String,

src/lot.rs

@@ -91,6 +91,9 @@ pub enum LotState {
     /// text1=domain
     QrAccount = 250,
 
+    /// text1=domain, text2=instance pattern
+    QrWebrtcInstance = 260,
+
     /// id=contact
     QrAddr = 320,
 

src/message.rs

@@ -6,15 +6,16 @@ use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Action};
 use crate::lot::{Lot, LotState, Meaning};
-use crate::mimeparser::SystemMessage;
+use crate::mimeparser::{FailureReport, SystemMessage};
 use crate::param::*;
 use crate::pgp::*;
 use crate::stock::StockMessage;
@@ -68,18 +69,38 @@ impl MsgId {
         self.0 == 0
     }
 
-    /// Whether the message ID is the special marker1 marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_marker1(self) -> bool {
-        self.0 == DC_MSG_ID_MARKER1
+    /// Returns message state.
+    pub async fn get_state(self, context: &Context) -> crate::sql::Result<MessageState> {
+        let result = context
+            .sql
+            .query_get_value_result("SELECT state FROM msgs WHERE id=?", paramsv![self])
+            .await?
+            .unwrap_or_default();
+        Ok(result)
     }
 
-    /// Whether the message ID is the special day marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_daymarker(self) -> bool {
-        self.0 == DC_MSG_ID_DAYMARKER
+    /// Returns true if the message needs to be moved from `folder`.
+    pub async fn needs_move(self, context: &Context, folder: &str) -> Result<bool, Error> {
+        if !context.get_config_bool(Config::MvboxMove).await {
+            return Ok(false);
+        }
+
+        if context.is_mvbox(folder).await {
+            return Ok(false);
+        }
+
+        let msg = Message::load_from_db(context, self).await?;
+
+        if msg.is_setupmessage() {
+            // do not move setup messages;
+            // there may be a non-delta device that wants to handle it
+            return Ok(false);
+        }
+
+        match msg.is_dc_message {
+            MessengerMessage::No => Ok(false),
+            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
+        }
     }
 
     /// Put message into trash chat and delete message text.
@@ -143,16 +164,7 @@ impl MsgId {
 
 impl std::fmt::Display for MsgId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // Would be nice if we could use match here, but no computed values in ranges.
-        if self.0 == DC_MSG_ID_MARKER1 {
-            write!(f, "Msg#Marker1")
-        } else if self.0 == DC_MSG_ID_DAYMARKER {
-            write!(f, "Msg#DayMarker")
-        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
-            write!(f, "Msg#UnknownSpecial")
-        } else {
-            write!(f, "Msg#{}", self.0)
-        }
+        write!(f, "Msg#{}", self.0)
     }
 }
 
@@ -246,6 +258,8 @@ pub struct Message {
     pub(crate) timestamp_sort: i64,
     pub(crate) timestamp_sent: i64,
     pub(crate) timestamp_rcvd: i64,
+    pub(crate) ephemeral_timer: u32,
+    pub(crate) ephemeral_timestamp: i64,
     pub(crate) text: Option<String>,
     pub(crate) rfc724_mid: String,
     pub(crate) in_reply_to: Option<String>,
@@ -255,6 +269,7 @@ pub struct Message {
     pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
+    pub(crate) error: String,
     pub(crate) param: Params,
 }
 
@@ -287,8 +302,11 @@ impl Message {
                     "    m.timestamp AS timestamp,",
                     "    m.timestamp_sent AS timestamp_sent,",
                     "    m.timestamp_rcvd AS timestamp_rcvd,",
+                    "    m.ephemeral_timer AS ephemeral_timer,",
+                    "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
+                    "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.txt AS txt,",
                     "    m.param AS param,",
@@ -314,8 +332,11 @@ impl Message {
                     msg.timestamp_sort = row.get("timestamp")?;
                     msg.timestamp_sent = row.get("timestamp_sent")?;
                     msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                    msg.ephemeral_timer = row.get("ephemeral_timer")?;
+                    msg.ephemeral_timestamp = row.get("ephemeral_timestamp")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
+                    msg.error = row.get("error")?;
                     msg.is_dc_message = row.get("msgrmsg")?;
 
                     let text;
@@ -390,7 +411,7 @@ impl Message {
                     }
 
                     if !self.id.is_unset() {
-                        self.save_param_to_disk(context).await;
+                        self.update_param(context).await;
                     }
                 }
             }
@@ -507,6 +528,14 @@ impl Message {
         self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
+    pub fn get_ephemeral_timer(&self) -> u32 {
+        self.ephemeral_timer
+    }
+
+    pub fn get_ephemeral_timestamp(&self) -> i64 {
+        self.ephemeral_timestamp
+    }
+
     pub async fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
         let mut ret = Lot::new();
 
@@ -571,6 +600,11 @@ impl Message {
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
+    pub fn is_system_message(&self) -> bool {
+        let cmd = self.param.get_cmd();
+        cmd != SystemMessage::Unknown
+    }
+
     /// Whether the message is still being created.
     ///
     /// Messages with attachments might be created before the
@@ -609,6 +643,76 @@ impl Message {
         None
     }
 
+    // add room to a webrtc_instance as defined by the corresponding config-value;
+    // the result may still be prefixed by the type
+    pub fn create_webrtc_instance(instance: &str, room: &str) -> String {
+        let (videochat_type, mut url) = Message::parse_webrtc_instance(instance);
+
+        // make sure, there is a scheme in the url
+        if !url.contains(':') {
+            url = format!("https://{}", url);
+        }
+
+        // add/replace room
+        let url = if url.contains("$ROOM") {
+            url.replace("$ROOM", &room)
+        } else {
+            // if there nothing that would separate the room, add a slash as a separator;
+            // this way, urls can be given as "https://meet.jit.si" as well as "https://meet.jit.si/"
+            let maybe_slash = if url.ends_with('/')
+                || url.ends_with('?')
+                || url.ends_with('#')
+                || url.ends_with('=')
+            {
+                ""
+            } else {
+                "/"
+            };
+            format!("{}{}{}", url, maybe_slash, room)
+        };
+
+        // re-add and normalize type
+        match videochat_type {
+            VideochatType::BasicWebrtc => format!("basicwebrtc:{}", url),
+            VideochatType::Jitsi => format!("jitsi:{}", url),
+            VideochatType::Unknown => url,
+        }
+    }
+
+    /// split a webrtc_instance as defined by the corresponding config-value into a type and a url
+    pub fn parse_webrtc_instance(instance: &str) -> (VideochatType, String) {
+        let instance: String = instance.split_whitespace().collect();
+        let mut split = instance.splitn(2, ':');
+        let type_str = split.next().unwrap_or_default().to_lowercase();
+        let url = split.next();
+        match type_str.as_str() {
+            "basicwebrtc" => (
+                VideochatType::BasicWebrtc,
+                url.unwrap_or_default().to_string(),
+            ),
+            "jitsi" => (VideochatType::Jitsi, url.unwrap_or_default().to_string()),
+            _ => (VideochatType::Unknown, instance.to_string()),
+        }
+    }
+
+    pub fn get_videochat_url(&self) -> Option<String> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).1);
+            }
+        }
+        None
+    }
+
+    pub fn get_videochat_type(&self) -> Option<VideochatType> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).0);
+            }
+        }
+        None
+    }
+
     pub fn set_text(&mut self, text: Option<String>) {
         self.text = text;
     }
@@ -643,10 +747,10 @@ impl Message {
         if duration > 0 {
             self.param.set_int(Param::Duration, duration);
         }
-        self.save_param_to_disk(context).await;
+        self.update_param(context).await;
     }
 
-    pub async fn save_param_to_disk(&mut self, context: &Context) -> bool {
+    pub async fn update_param(&mut self, context: &Context) -> bool {
         context
             .sql
             .execute(
@@ -763,9 +867,10 @@ impl From<MessageState> for LotState {
 impl MessageState {
     pub fn can_fail(self) -> bool {
         match self {
-            MessageState::OutPreparing | MessageState::OutPending | MessageState::OutDelivered => {
-                true
-            }
+            MessageState::OutPreparing
+            | MessageState::OutPending
+            | MessageState::OutDelivered
+            | MessageState::OutMdnRcvd => true, // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
             _ => false,
         }
     }
@@ -887,6 +992,17 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         ret += "\n";
     }
 
+    if msg.ephemeral_timer != 0 {
+        ret += &format!("Ephemeral timer: {}\n", msg.ephemeral_timer);
+    }
+
+    if msg.ephemeral_timestamp != 0 {
+        ret += &format!(
+            "Expires: {}\n",
+            dc_timestamp_to_str(msg.ephemeral_timestamp)
+        );
+    }
+
     if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
@@ -937,8 +1053,9 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
     }
 
     ret += "\n";
-    if let Some(err) = msg.param.get(Param::Error) {
-        ret += &format!("Error: {}", err)
+
+    if !msg.error.is_empty() {
+        ret += &format!("Error: {}", msg.error);
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -979,18 +1096,30 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     let extension: &str = &path.extension()?.to_str()?.to_lowercase();
     let info = match extension {
-        "mp3" => (Viewtype::Audio, "audio/mpeg"),
+        "3gp" => (Viewtype::Video, "video/3gpp"),
         "aac" => (Viewtype::Audio, "audio/aac"),
-        "mp4" => (Viewtype::Video, "video/mp4"),
-        "webm" => (Viewtype::Video, "video/webm"),
-        "jpg" => (Viewtype::Image, "image/jpeg"),
+        "avi" => (Viewtype::Video, "video/x-msvideo"),
+        "flac" => (Viewtype::Audio, "audio/flac"),
+        "gif" => (Viewtype::Gif, "image/gif"),
         "jpeg" => (Viewtype::Image, "image/jpeg"),
         "jpe" => (Viewtype::Image, "image/jpeg"),
+        "jpg" => (Viewtype::Image, "image/jpeg"),
+        "mov" => (Viewtype::Video, "video/quicktime"),
+        "mp3" => (Viewtype::Audio, "audio/mpeg"),
+        "mp4" => (Viewtype::Video, "video/mp4"),
+        "oga" => (Viewtype::Audio, "audio/ogg"),
+        "ogg" => (Viewtype::Audio, "audio/ogg"),
+        "ogv" => (Viewtype::Video, "video/ogg"),
+        "opus" => (Viewtype::Audio, "audio/ogg"),
         "png" => (Viewtype::Image, "image/png"),
-        "webp" => (Viewtype::Image, "image/webp"),
-        "gif" => (Viewtype::Gif, "image/gif"),
-        "vcf" => (Viewtype::File, "text/vcard"),
+        "spx" => (Viewtype::Audio, "audio/ogg"), // Ogg Speex Profile
+        "svg" => (Viewtype::Image, "image/svg+xml"),
+        "tgs" => (Viewtype::Sticker, "application/x-tgsticker"),
         "vcard" => (Viewtype::File, "text/vcard"),
+        "vcf" => (Viewtype::File, "text/vcard"),
+        "webm" => (Viewtype::Video, "video/webm"),
+        "webp" => (Viewtype::Image, "image/webp"),
+        "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
         _ => {
             return None;
         }
@@ -1027,7 +1156,7 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
     }
 
     if !msg_ids.is_empty() {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -1091,6 +1220,14 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
     let mut send_event = false;
 
     for (id, curr_state, curr_blocked) in msgs.into_iter() {
+        if let Err(err) = id.start_ephemeral_timer(context).await {
+            error!(
+                context,
+                "Failed to start ephemeral timer for message {}: {}", id, err
+            );
+            continue;
+        }
+
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, id, MessageState::InSeen).await;
@@ -1110,7 +1247,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
     }
 
     if send_event {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -1147,7 +1284,7 @@ pub async fn star_msgs(context: &Context, msg_ids: Vec<MsgId>, star: bool) -> bo
         .is_ok()
 }
 
-/// Returns a summary test.
+/// Returns a summary text.
 pub async fn get_summarytext_by_raw(
     viewtype: Viewtype,
     text: Option<impl AsRef<str>>,
@@ -1191,6 +1328,13 @@ pub async fn get_summarytext_by_raw(
                 format!("{} – {}", label, file_name)
             }
         }
+        Viewtype::VideochatInvitation => {
+            append_text = false;
+            context
+                .stock_str(StockMessage::VideochatInvitation)
+                .await
+                .into_owned()
+        }
         _ => {
             if param.get_cmd() != SystemMessage::LocationOnly {
                 "".to_string()
@@ -1251,39 +1395,44 @@ pub async fn exists(context: &Context, msg_id: MsgId) -> bool {
 
 pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id).await {
+        let error = error.map(|e| e.as_ref().to_string()).unwrap_or_default();
         if msg.state.can_fail() {
             msg.state = MessageState::OutFailed;
-        }
-        if let Some(error) = error {
-            msg.param.set(Param::Error, error.as_ref());
-            warn!(context, "Message failed: {}", error.as_ref());
+            warn!(context, "{} failed: {}", msg_id, error);
+        } else {
+            warn!(
+                context,
+                "{} seems to have failed ({}), but state is {}", msg_id, error, msg.state
+            )
         }
 
-        if context
+        match context
             .sql
             .execute(
-                "UPDATE msgs SET state=?, param=? WHERE id=?;",
-                paramsv![msg.state, msg.param.to_string(), msg_id],
+                "UPDATE msgs SET state=?, error=? WHERE id=?;",
+                paramsv![msg.state, error, msg_id],
             )
             .await
-            .is_ok()
         {
-            context.emit_event(Event::MsgFailed {
+            Ok(_) => context.emit_event(EventType::MsgFailed {
                 chat_id: msg.chat_id,
                 msg_id,
-            });
+            }),
+            Err(e) => {
+                warn!(context, "{:?}", e);
+            }
         }
     }
 }
 
 /// returns Some if an event should be send
-pub async fn mdn_from_ext(
+pub async fn handle_mdn(
     context: &Context,
     from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Option<(ChatId, MsgId)> {
-    if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
+    if from_id <= DC_CONTACT_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
         return None;
     }
 
@@ -1318,10 +1467,10 @@ pub async fn mdn_from_ext(
     if let Ok((msg_id, chat_id, chat_type, msg_state)) = res {
         let mut read_by_all = false;
 
-        // if already marked as MDNS_RCVD msgstate_can_fail() returns false.
-        // however, it is important, that ret_msg_id is set above as this
-        // will allow the caller eg. to move the message away
-        if msg_state.can_fail() {
+        if msg_state == MessageState::OutPreparing
+            || msg_state == MessageState::OutPending
+            || msg_state == MessageState::OutDelivered
+        {
             let mdn_already_in_table = context
                 .sql
                 .exists(
@@ -1384,6 +1533,69 @@ pub async fn mdn_from_ext(
     None
 }
 
+/// Marks a message as failed after an ndn (non-delivery-notification) arrived.
+/// Where appropriate, also adds an info message telling the user which of the recipients of a group message failed.
+pub(crate) async fn handle_ndn(
+    context: &Context,
+    failed: &FailureReport,
+    error: Option<impl AsRef<str>>,
+) {
+    if failed.rfc724_mid.is_empty() {
+        return;
+    }
+
+    let res = context
+        .sql
+        .query_row(
+            concat!(
+                "SELECT",
+                "    m.id AS msg_id,",
+                "    c.id AS chat_id,",
+                "    c.type AS type",
+                " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+                " WHERE rfc724_mid=? AND from_id=1",
+            ),
+            paramsv![failed.rfc724_mid],
+            |row| {
+                Ok((
+                    row.get::<_, MsgId>("msg_id")?,
+                    row.get::<_, ChatId>("chat_id")?,
+                    row.get::<_, Chattype>("type")?,
+                ))
+            },
+        )
+        .await;
+    if let Err(ref err) = res {
+        info!(context, "Failed to select NDN {:?}", err);
+    }
+
+    if let Ok((msg_id, chat_id, chat_type)) = res {
+        set_msg_failed(context, msg_id, error).await;
+
+        if chat_type == Chattype::Group || chat_type == Chattype::VerifiedGroup {
+            if let Some(failed_recipient) = &failed.failed_recipient {
+                let contact_id =
+                    Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
+                if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+                    // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
+                    chat::add_info_msg(
+                        context,
+                        chat_id,
+                        context
+                            .stock_string_repl_str(
+                                StockMessage::FailedSendingTo,
+                                contact.get_display_name(),
+                            )
+                            .await,
+                    )
+                    .await;
+                    context.emit_event(EventType::ChatModified(chat_id));
+                }
+            }
+        }
+    }
+}
+
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
 pub async fn get_real_msg_cnt(context: &Context) -> i32 {
     match context
@@ -1558,6 +1770,7 @@ pub async fn dc_empty_server(context: &Context, flags: u32) {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chat::ChatItem;
     use crate::test_utils as test;
 
     #[test]
@@ -1572,7 +1785,7 @@ mod tests {
     async fn test_prepare_message_and_send() {
         use crate::config::Config;
 
-        let d = test::dummy_context().await;
+        let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
         let contact = Contact::create(ctx, "", "dest@example.com")
@@ -1596,7 +1809,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_summarytext_by_raw() {
-        let d = test::dummy_context().await;
+        let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
         let some_text = Some("bla bla".to_string());
@@ -1681,4 +1894,91 @@ mod tests {
             "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
         );
     }
+
+    #[async_std::test]
+    async fn test_parse_webrtc_instance() {
+        let (webrtc_type, url) = Message::parse_webrtc_instance("basicwebrtc:https://foo/bar");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "https://foo/bar");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("bAsIcwEbrTc:url");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "url");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("https://foo/bar?key=val#key=val");
+        assert_eq!(webrtc_type, VideochatType::Unknown);
+        assert_eq!(url, "https://foo/bar?key=val#key=val");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("jitsi:https://j.si/foo");
+        assert_eq!(webrtc_type, VideochatType::Jitsi);
+        assert_eq!(url, "https://j.si/foo");
+    }
+
+    #[async_std::test]
+    async fn test_create_webrtc_instance() {
+        // webrtc_instance may come from an input field of the ui, be pretty tolerant on input
+        let instance = Message::create_webrtc_instance("https://meet.jit.si/", "123");
+        assert_eq!(instance, "https://meet.jit.si/123");
+
+        let instance = Message::create_webrtc_instance("https://meet.jit.si", "456");
+        assert_eq!(instance, "https://meet.jit.si/456");
+
+        let instance = Message::create_webrtc_instance("meet.jit.si", "789");
+        assert_eq!(instance, "https://meet.jit.si/789");
+
+        let instance = Message::create_webrtc_instance("bla.foo?", "123");
+        assert_eq!(instance, "https://bla.foo?123");
+
+        let instance = Message::create_webrtc_instance("jitsi:bla.foo#", "456");
+        assert_eq!(instance, "jitsi:https://bla.foo#456");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room=", "789");
+        assert_eq!(instance, "https://bla.foo#room=789");
+
+        let instance = Message::create_webrtc_instance("https://bla.foo#room", "123");
+        assert_eq!(instance, "https://bla.foo#room/123");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room$ROOM", "123");
+        assert_eq!(instance, "https://bla.foo#room123");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room=$ROOM&after=cont", "234");
+        assert_eq!(instance, "https://bla.foo#room=234&after=cont");
+
+        let instance = Message::create_webrtc_instance("  meet.jit .si ", "789");
+        assert_eq!(instance, "https://meet.jit.si/789");
+
+        let instance = Message::create_webrtc_instance(" basicwebrtc: basic . stuff\n ", "12345ab");
+        assert_eq!(instance, "basicwebrtc:https://basic.stuff/12345ab");
+    }
+
+    #[async_std::test]
+    async fn test_get_width_height() {
+        let t = test::TestContext::new().await;
+
+        // test that get_width() and get_height() are returning some dimensions for images;
+        // (as the device-chat contains a welcome-images, we check that)
+        t.ctx.update_device_chats().await.ok();
+        let (device_chat_id, _) =
+            chat::create_or_lookup_by_contact_id(&t.ctx, DC_CONTACT_ID_DEVICE, Blocked::Not)
+                .await
+                .unwrap();
+
+        let mut has_image = false;
+        let chatitems = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        for chatitem in chatitems {
+            if let ChatItem::Message { msg_id } = chatitem {
+                if let Ok(msg) = Message::load_from_db(&t.ctx, msg_id.clone()).await {
+                    if msg.get_viewtype() == Viewtype::Image {
+                        has_image = true;
+                        // just check that width/height are inside some reasonable ranges
+                        assert!(msg.get_width() > 100);
+                        assert!(msg.get_height() > 100);
+                        assert!(msg.get_width() < 4000);
+                        assert!(msg.get_height() < 4000);
+                    }
+                }
+            }
+        }
+        assert!(has_image);
+    }
 }

src/mimefactory.rs

@@ -9,6 +9,7 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
 use crate::location;
 use crate::message::{self, Message};
@@ -112,12 +113,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 )
                 .await?;
 
-            let command = msg.param.get_cmd();
-
-            if command != SystemMessage::AutocryptSetupMessage
-                && command != SystemMessage::SecurejoinMessage
-                && context.get_config_bool(Config::MdnsEnabled).await
-            {
+            if !msg.is_system_message() && context.get_config_bool(Config::MdnsEnabled).await {
                 req_mdn = true;
             }
         }
@@ -225,7 +221,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .filter(|(_, addr)| addr != &self_addr)
         {
             res.push((
-                Peerstate::from_addr(self.context, addr).await,
+                Peerstate::from_addr(self.context, addr).await?,
                 addr.as_str(),
             ));
         }
@@ -457,21 +453,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         unprotected_headers.push(Header::new("Date".into(), date));
 
-        let os_name = &self.context.os_name;
-        let os_part = os_name
-            .as_ref()
-            .map(|s| format!("/{}", s))
-            .unwrap_or_default();
-        let version = get_version_str();
-
-        // Add a X-Mailer header.
-        // This is only informational for debugging and may be removed in the release.
-        // We do not rely on this header as it may be removed by MTAs.
-
-        unprotected_headers.push(Header::new(
-            "X-Mailer".into(),
-            format!("Delta Chat Core {}{}", version, os_part),
-        ));
         unprotected_headers.push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
 
         if let Loaded::MDN { .. } = self.loaded {
@@ -498,7 +479,16 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(self.context).await?;
 
-        let subject = encode_words(&subject_str);
+        let subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str
+        } else {
+            encode_words(&subject_str)
+        };
 
         let mut message = match self.loaded {
             Loaded::Message { .. } => {
@@ -526,6 +516,14 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
         };
 
+        let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(self.context).await?;
+        if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
+            protected_headers.push(Header::new(
+                "Ephemeral-Timer".to_string(),
+                duration.to_string(),
+            ));
+        }
+
         // we could also store the message-id in the protected headers
         // which would probably help to survive providers like
         // Outlook.com or hotmail which mangle the Message-ID.
@@ -776,6 +774,26 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     "location-streaming-enabled".into(),
                 ));
             }
+            SystemMessage::EphemeralTimerChanged => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "ephemeral-timer-changed".to_string(),
+                ));
+            }
+            SystemMessage::LocationOnly => {
+                // This should prevent automatic replies,
+                // such as non-delivery reports.
+                //
+                // See https://tools.ietf.org/html/rfc3834
+                //
+                // Adding this header without encryption leaks some
+                // information about the message contents, but it can
+                // already be easily guessed from message timing and size.
+                unprotected_headers.push(Header::new(
+                    "Auto-Submitted".to_string(),
+                    "auto-generated".to_string(),
+                ));
+            }
             SystemMessage::AutocryptSetupMessage => {
                 unprotected_headers
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
@@ -837,6 +855,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if self.msg.viewtype == Viewtype::Sticker {
             protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
+        } else if self.msg.viewtype == Viewtype::VideochatInvitation {
+            protected_headers.push(Header::new(
+                "Chat-Content".into(),
+                "videochat-invitation".into(),
+            ));
+            protected_headers.push(Header::new(
+                "Chat-Webrtc-Room".into(),
+                self.msg
+                    .param
+                    .get(Param::WebrtcRoom)
+                    .unwrap_or_default()
+                    .into(),
+            ));
         }
 
         if self.msg.viewtype == Viewtype::Voice
@@ -1210,7 +1241,6 @@ mod tests {
     use crate::chatlist::Chatlist;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::mimeparser::*;
-    use crate::test_utils::configured_offline_context;
     use crate::test_utils::TestContext;
 
     #[test]
@@ -1300,10 +1330,10 @@ mod tests {
         // 1.: Receive a mail from an MUA or Delta Chat
         assert_eq!(
             msg_to_subject_str(
-                b"From: Bob <bob@example.org>\n\
-                To: alice@example.org\n\
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
                 Subject: Antw: Chat: hello\n\
-                Message-ID: <2222@example.org>\n\
+                Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                 \n\
                 hello\n"
@@ -1314,10 +1344,10 @@ mod tests {
 
         assert_eq!(
             msg_to_subject_str(
-                b"From: Bob <bob@example.org>\n\
-                To: alice@example.org\n\
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
                 Subject: Infos: 42\n\
-                Message-ID: <2222@example.org>\n\
+                Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                 \n\
                 hello\n"
@@ -1329,11 +1359,11 @@ mod tests {
         // 2. Receive a message from Delta Chat when we did not send any messages before
         assert_eq!(
             msg_to_subject_str(
-                b"From: Charlie <charlie@example.org>\n\
-                To: alice@example.org\n\
+                b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
-                Message-ID: <2223@example.org>\n\
+                Message-ID: <2223@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                 \n\
                 hello\n"
@@ -1343,10 +1373,11 @@ mod tests {
         );
 
         // 3. Send the first message to a new contact
-        let t = configured_offline_context().await;
-        assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
+        let t = TestContext::new_alice().await;
 
-        let t = configured_offline_context().await;
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+
+        let t = TestContext::new_alice().await;
         t.ctx
             .set_config(Config::Displayname, Some("Alice"))
             .await
@@ -1355,11 +1386,11 @@ mod tests {
 
         // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
         msg_to_subject_str(
-            "From: Charlie <charlie@example.org>\n\
-            To: alice@example.org\n\
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
             Subject: äääää\n\
             Chat-Version: 1.0\n\
-            Message-ID: <2893@example.org>\n\
+            Message-ID: <2893@example.com>\n\
             Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
             \n\
             hello\n"
@@ -1368,22 +1399,70 @@ mod tests {
         .await;
 
         msg_to_subject_str(
-            "From: Charlie <charlie@example.org>\n\
-            To: alice@example.org\n\
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
             Subject: aäääää\n\
             Chat-Version: 1.0\n\
-            Message-ID: <2893@example.org>\n\
+            Message-ID: <2893@example.com>\n\
             Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
             \n\
             hello\n"
                 .as_bytes(),
         )
         .await;
+
+        // 5. Receive an mdn (read receipt) and make sure the mdn's subject is not used
+        let t = TestContext::new_alice().await;
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@example.com\n\
+            To: Charlie <charlie@example.com>\n\
+            Subject: Hello, Charlie\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        let new_msg = incoming_msg_to_reply_msg(b"From: charlie@example.com\n\
+                 To: alice@example.com\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;charlie@example.com\n\
+                 Final-Recipient: rfc822;charlie@example.com\n\
+                 Original-Message-ID: <2893@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n", &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        // The subject string should not be "Re: message opened"
+        assert_eq!("Re: Hello, Charlie", mf.subject_str().await);
     }
 
     async fn first_subject_str(t: TestContext) -> String {
         let contact_id =
-            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.org", Origin::ManuallyCreated)
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.com", Origin::ManuallyCreated)
                 .await
                 .unwrap()
                 .0;
@@ -1407,7 +1486,7 @@ mod tests {
     }
 
     async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
         let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
             .await
@@ -1415,7 +1494,7 @@ mod tests {
         mf.subject_str().await
     }
 
-    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    // Creates a `Message` that replies "Hi" to the incoming email in `imf_raw`.
     async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
         context
             .set_config(Config::ShowEmails, Some("2"))
@@ -1445,15 +1524,15 @@ mod tests {
     #[async_std::test]
     // This test could still be extended
     async fn test_render_reply() {
-        let t = configured_offline_context().await;
+        let t = TestContext::new_alice().await;
         let context = &t.ctx;
 
         let msg = incoming_msg_to_reply_msg(
-            b"From: Charlie <charlie@example.org>\n\
-                To: alice@example.org\n\
+            b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
-                Message-ID: <2223@example.org>\n\
+                Message-ID: <2223@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                 \n\
                 hello\n",
@@ -1464,7 +1543,7 @@ mod tests {
         let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
 
         let recipients = mimefactory.recipients();
-        assert_eq!(recipients, vec!["charlie@example.org"]);
+        assert_eq!(recipients, vec!["charlie@example.com"]);
 
         let rendered_msg = mimefactory.render().await.unwrap();
 

src/mimeparser.rs

@@ -3,6 +3,7 @@ use std::future::Future;
 use std::pin::Pin;
 
 use deltachat_derive::{FromSql, ToSql};
+use lazy_static::lazy_static;
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 
@@ -15,14 +16,13 @@ use crate::dc_tools::*;
 use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::error::{bail, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
 use crate::param::*;
 use crate::peerstate::Peerstate;
-use crate::securejoin::handle_degrade_event;
 use crate::simplify::*;
 use crate::stock::StockMessage;
 
@@ -45,7 +45,14 @@ pub struct MimeMessage {
     pub from: Vec<SingleInfo>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
+
+    /// Set of valid signature fingerprints if a message is an
+    /// Autocrypt encrypted and signed message.
+    ///
+    /// If a message is not encrypted or the signature is not valid,
+    /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
+
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
@@ -53,7 +60,8 @@ pub struct MimeMessage {
     pub message_kml: Option<location::Kml>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
-    pub(crate) reports: Vec<Report>,
+    pub(crate) mdn_reports: Vec<Report>,
+    pub(crate) failure_report: Option<FailureReport>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -74,6 +82,9 @@ pub enum SystemMessage {
     SecurejoinMessage = 7,
     LocationStreamingEnabled = 8,
     LocationOnly = 9,
+
+    /// Chat ephemeral message timer is changed.
+    EphemeralTimerChanged = 10,
 }
 
 impl Default for SystemMessage {
@@ -116,53 +127,74 @@ impl MimeMessage {
         let mail_raw;
         let mut gossipped_addr = Default::default();
 
-        let (mail, signatures) = match e2ee::try_decrypt(context, &mail, message_time).await {
-            Ok((raw, signatures)) => {
-                if let Some(raw) = raw {
-                    // Valid autocrypt message, encrypted
-                    mail_raw = raw;
-                    let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
-                    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
-                        info!(context, "decrypted message mime-body:");
-                        println!("{}", String::from_utf8_lossy(&mail_raw));
-                    }
+        let (mail, signatures, warn_empty_signature) =
+            match e2ee::try_decrypt(context, &mail, message_time).await {
+                Ok((raw, signatures)) => {
+                    if let Some(raw) = raw {
+                        // Encrypted, but maybe unsigned message. Only if
+                        // `signatures` set is non-empty, it is a valid
+                        // autocrypt message.
 
-                    // Handle any gossip headers if the mail was encrypted.  See section
-                    // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
-                    let gossip_headers = decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
-                    gossipped_addr =
-                        update_gossip_peerstates(context, message_time, &mail, gossip_headers)
+                        mail_raw = raw;
+                        let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
+                        if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+                            info!(context, "decrypted message mime-body:");
+                            println!("{}", String::from_utf8_lossy(&mail_raw));
+                        }
+
+                        // Handle any gossip headers if the mail was encrypted.  See section
+                        // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
+                        // but only if the mail was correctly signed:
+                        if !signatures.is_empty() {
+                            let gossip_headers =
+                                decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
+                            gossipped_addr = update_gossip_peerstates(
+                                context,
+                                message_time,
+                                &mail,
+                                gossip_headers,
+                            )
                             .await?;
+                        }
 
-                    // let known protected headers from the decrypted
-                    // part override the unencrypted top-level
-                    MimeMessage::merge_headers(
-                        context,
-                        &mut headers,
-                        &mut recipients,
-                        &mut from,
-                        &mut chat_disposition_notification_to,
-                        &decrypted_mail.headers,
-                    );
+                        // let known protected headers from the decrypted
+                        // part override the unencrypted top-level
 
-                    (decrypted_mail, signatures)
-                } else {
-                    // Message was not encrypted
-                    (mail, signatures)
+                        // Signature was checked for original From, so we
+                        // do not allow overriding it.
+                        let mut throwaway_from = from.clone();
+
+                        // We do not want to allow unencrypted subject in encrypted emails because the user might falsely think that the subject is safe.
+                        // See https://github.com/deltachat/deltachat-core-rust/issues/1790.
+                        headers.remove("subject");
+
+                        MimeMessage::merge_headers(
+                            context,
+                            &mut headers,
+                            &mut recipients,
+                            &mut throwaway_from,
+                            &mut chat_disposition_notification_to,
+                            &decrypted_mail.headers,
+                        );
+
+                        (decrypted_mail, signatures, true)
+                    } else {
+                        // Message was not encrypted
+                        (mail, signatures, false)
+                    }
                 }
-            }
-            Err(err) => {
-                // continue with the current, still encrypted, mime tree.
-                // unencrypted parts will be replaced by an error message
-                // that is added as "the message" to the chat then.
-                //
-                // if we just return here, the header is missing
-                // and the caller cannot display the message
-                // and try to assign the message to a chat
-                warn!(context, "decryption failed: {}", err);
-                (mail, Default::default())
-            }
-        };
+                Err(err) => {
+                    // continue with the current, still encrypted, mime tree.
+                    // unencrypted parts will be replaced by an error message
+                    // that is added as "the message" to the chat then.
+                    //
+                    // if we just return here, the header is missing
+                    // and the caller cannot display the message
+                    // and try to assign the message to a chat
+                    warn!(context, "decryption failed: {}", err);
+                    (mail, Default::default(), true)
+                }
+            };
 
         let mut parser = MimeMessage {
             parts: Vec::new(),
@@ -176,16 +208,24 @@ impl MimeMessage {
             signatures,
             gossipped_addr,
             is_forwarded: false,
-            reports: Vec::new(),
+            mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
             user_avatar: None,
             group_avatar: None,
+            failure_report: None,
         };
         parser.parse_mime_recursive(context, &mail).await?;
+        parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
+        if warn_empty_signature && parser.signatures.is_empty() {
+            for part in parser.parts.iter_mut() {
+                part.error = "No valid signature".to_string();
+            }
+        }
+
         Ok(parser)
     }
 
@@ -210,6 +250,8 @@ impl MimeMessage {
         } else if let Some(value) = self.get(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
+            } else if value == "ephemeral-timer-changed" {
+                self.is_system_message = SystemMessage::EphemeralTimerChanged;
             }
         }
         Ok(())
@@ -226,10 +268,24 @@ impl MimeMessage {
         }
     }
 
+    fn parse_videochat_headers(&mut self) {
+        if let Some(value) = self.get(HeaderDef::ChatContent).cloned() {
+            if value == "videochat-invitation" {
+                let instance = self.get(HeaderDef::ChatWebrtcRoom).cloned();
+                if let Some(part) = self.parts.first_mut() {
+                    part.typ = Viewtype::VideochatInvitation;
+                    part.param
+                        .set(Param::WebrtcRoom, instance.unwrap_or_default());
+                }
+            }
+        }
+    }
+
     /// Squashes mutlipart chat messages with attachment into single-part messages.
     ///
     /// Delta Chat sends attachments, such as images, in two-part messages, with the first message
     /// containing an explanation. If such a message is detected, first part can be safely dropped.
+    #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
             let need_drop = {
@@ -253,7 +309,8 @@ impl MimeMessage {
                 self.parts[0].msg = "".to_string();
 
                 // swap new with old
-                std::mem::replace(&mut self.parts[0], filepart);
+                self.parts.push(filepart); // push to the end
+                let _ = self.parts.swap_remove(0); // drops first element, replacing it with the last one in O(1)
             }
         }
     }
@@ -262,22 +319,21 @@ impl MimeMessage {
     fn parse_attachments(&mut self) {
         // Attachment messages should be squashed into a single part
         // before calling this function.
-        if self.parts.len() == 1 {
-            if self.parts[0].typ == Viewtype::Audio
-                && self.get(HeaderDef::ChatVoiceMessage).is_some()
-            {
-                let part_mut = &mut self.parts[0];
-                part_mut.typ = Viewtype::Voice;
+        if self.parts.len() != 1 {
+            return;
+        }
+
+        if let Some(mut part) = self.parts.pop() {
+            if part.typ == Viewtype::Audio && self.get(HeaderDef::ChatVoiceMessage).is_some() {
+                part.typ = Viewtype::Voice;
             }
-            if self.parts[0].typ == Viewtype::Image {
+            if part.typ == Viewtype::Image {
                 if let Some(value) = self.get(HeaderDef::ChatContent) {
                     if value == "sticker" {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.typ = Viewtype::Sticker;
+                        part.typ = Viewtype::Sticker;
                     }
                 }
             }
-            let part = &self.parts[0];
             if part.typ == Viewtype::Audio
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
@@ -285,17 +341,19 @@ impl MimeMessage {
                 if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.param.set_int(Param::Duration, duration_ms);
+                        part.param.set_int(Param::Duration, duration_ms);
                     }
                 }
             }
+
+            self.parts.push(part);
         }
     }
 
     fn parse_headers(&mut self, context: &Context) -> Result<()> {
         self.parse_system_message_headers(context)?;
         self.parse_avatar_headers();
+        self.parse_videochat_headers();
         self.squash_attachment_parts();
 
         if let Some(ref subject) = self.get_subject() {
@@ -311,12 +369,11 @@ impl MimeMessage {
                 }
             }
             if prepend_subject {
-                let subj = if let Some(n) = subject.find('[') {
-                    &subject[0..n]
-                } else {
-                    subject
-                }
-                .trim();
+                let subj = subject
+                    .find('[')
+                    .and_then(|n| subject.get(..n))
+                    .unwrap_or(subject)
+                    .trim();
 
                 if !subj.is_empty() {
                     for part in self.parts.iter_mut() {
@@ -353,7 +410,7 @@ impl MimeMessage {
         // just have send a message in the subject with an empty body.
         // Besides, we want to show something in case our incoming-processing
         // failed to properly handle an incoming message.
-        if self.parts.is_empty() && self.reports.is_empty() {
+        if self.parts.is_empty() && self.mdn_reports.is_empty() {
             let mut part = Part::default();
             part.typ = Viewtype::Text;
 
@@ -374,8 +431,7 @@ impl MimeMessage {
             Some(AvatarAction::Delete)
         } else {
             let mut i = 0;
-            while i != self.parts.len() {
-                let part = &mut self.parts[i];
+            while let Some(part) = self.parts.get_mut(i) {
                 if let Some(part_filename) = &part.org_filename {
                     if part_filename == &header_value {
                         if let Some(blob) = part.param.get(Param::File) {
@@ -392,6 +448,11 @@ impl MimeMessage {
         }
     }
 
+    /// Returns true if the message was encrypted as defined in
+    /// Autocrypt standard.
+    ///
+    /// This means the message was both encrypted and signed with a
+    /// valid signature.
     pub fn was_encrypted(&self) -> bool {
         !self.signatures.is_empty()
     }
@@ -527,7 +588,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
-                part.param.set(Param::Error, "Decryption failed");
+                part.error = "Decryption failed".to_string();
 
                 self.parts.push(part);
 
@@ -539,7 +600,7 @@ impl MimeMessage {
                 contains exactly two body parts.  The first body
                 part is the body part over which the digital signature was created [...]
                 The second body part contains the control information necessary to
-                verify the digital signature." We simpliy take the first body part and
+                verify the digital signature." We simply take the first body part and
                 skip the rest.  (see
                 https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
                 for background information why we use encrypted+signed) */
@@ -550,10 +611,10 @@ impl MimeMessage {
             (mime::MULTIPART, "report") => {
                 /* RFC 6522: the first part is for humans, the second for machines */
                 if mail.subparts.len() >= 2 {
-                    if let Some(report_type) = mail.ctype.params.get("report-type") {
-                        if report_type == "disposition-notification" {
+                    match mail.ctype.params.get("report-type").map(|s| s as &str) {
+                        Some("disposition-notification") => {
                             if let Some(report) = self.process_report(context, mail)? {
-                                self.reports.push(report);
+                                self.mdn_reports.push(report);
                             }
 
                             // Add MDN part so we can track it, avoid
@@ -565,9 +626,21 @@ impl MimeMessage {
                             self.parts.push(part);
 
                             any_part_added = true;
-                        } else {
-                            /* eg. `report-type=delivery-status`;
-                            maybe we should show them as a little error icon */
+                        }
+                        // Some providers, e.g. Tiscali, forget to set the report-type. So, if it's None, assume that it might be delivery-status
+                        Some("delivery-status") | None => {
+                            if let Some(report) = self.process_delivery_status(context, mail)? {
+                                self.failure_report = Some(report);
+                            }
+
+                            // Add all parts (we need another part, preferrably text/plain, to show as an error message)
+                            for cur_data in mail.subparts.iter() {
+                                if self.parse_mime_recursive(context, cur_data).await? {
+                                    any_part_added = true;
+                                }
+                            }
+                        }
+                        Some(_) => {
                             if let Some(first) = mail.subparts.iter().next() {
                                 any_part_added = self.parse_mime_recursive(context, first).await?;
                             }
@@ -748,16 +821,11 @@ impl MimeMessage {
     }
 
     pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
-        if self.parts.is_empty() {
-            return;
+        if let Some(part) = self.parts.first_mut() {
+            part.typ = Viewtype::Text;
+            part.msg = format!("[{}]", error_msg.as_ref());
+            self.parts.truncate(1);
         }
-
-        let part = &mut self.parts[0];
-        part.typ = Viewtype::Text;
-        part.msg = format!("[{}]", error_msg.as_ref());
-        self.parts.truncate(1);
-
-        assert_eq!(self.parts.len(), 1);
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
@@ -808,7 +876,11 @@ impl MimeMessage {
         report: &mailparse::ParsedMail<'_>,
     ) -> Result<Option<Report>> {
         // parse as mailheaders
-        let report_body = report.subparts[1].get_body_raw()?;
+        let report_body = if let Some(subpart) = report.subparts.get(1) {
+            subpart.get_body_raw()?
+        } else {
+            bail!("Report does not have second MIME part");
+        };
         let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
         // must be present
@@ -840,24 +912,115 @@ impl MimeMessage {
         Ok(None)
     }
 
-    /// Handle reports (only MDNs for now)
-    pub async fn handle_reports(&self, context: &Context, from_id: u32, sent_timestamp: i64) {
-        if self.reports.is_empty() {
-            return;
+    fn process_delivery_status(
+        &self,
+        context: &Context,
+        report: &mailparse::ParsedMail<'_>,
+    ) -> Result<Option<FailureReport>> {
+        // parse as mailheaders
+        if let Some(original_msg) = report
+            .subparts
+            .iter()
+            .find(|p| p.ctype.mimetype.contains("rfc822") || p.ctype.mimetype == "message/global")
+        {
+            let report_body = original_msg.get_body_raw()?;
+            let (report_fields, _) = mailparse::parse_headers(&report_body)?;
+
+            if let Some(original_message_id) = report_fields
+                .get_header_value(HeaderDef::MessageId)
+                .and_then(|v| parse_message_id(&v).ok())
+            {
+                let mut to_list = get_all_addresses_from_header(&report.headers, |header_key| {
+                    header_key == "x-failed-recipients"
+                });
+                let to = if to_list.len() == 1 {
+                    Some(to_list.pop().unwrap())
+                } else {
+                    None // We do not know which recipient failed
+                };
+
+                return Ok(Some(FailureReport {
+                    rfc724_mid: original_message_id,
+                    failed_recipient: to.map(|s| s.addr),
+                }));
+            }
+
+            warn!(
+                context,
+                "ignoring unknown ndn-notification, Message-Id: {:?}",
+                report_fields.get_header_value(HeaderDef::MessageId)
+            );
         }
 
-        for report in &self.reports {
+        Ok(None)
+    }
+
+    /// Some providers like GMX and Yahoo do not send standard NDNs (Non Delivery notifications).
+    /// If you improve heuristics here you might also have to change prefetch_should_download() in imap/mod.rs.
+    /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
+    #[allow(clippy::indexing_slicing)]
+    async fn heuristically_parse_ndn(&mut self, context: &Context) -> Option<()> {
+        let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
+            let from = from.to_ascii_lowercase();
+            from.contains("mailer-daemon") || from.contains("mail-daemon")
+        } else {
+            false
+        };
+        if maybe_ndn && self.failure_report.is_none() {
+            lazy_static! {
+                static ref RE: regex::Regex = regex::Regex::new(r"Message-ID:(.*)").unwrap();
+            }
+            for captures in self
+                .parts
+                .iter()
+                .filter_map(|part| part.msg_raw.as_ref())
+                .flat_map(|part| part.lines())
+                .filter_map(|line| RE.captures(line))
+            {
+                if let Ok(original_message_id) = parse_message_id(&captures[1]) {
+                    if let Ok(Some(_)) =
+                        message::rfc724_mid_exists(context, &original_message_id).await
+                    {
+                        self.failure_report = Some(FailureReport {
+                            rfc724_mid: original_message_id,
+                            failed_recipient: None,
+                        })
+                    }
+                }
+            }
+        }
+        None // Always return None, we just return anything so that we can use the '?' operator.
+    }
+
+    /// Handle reports
+    /// (MDNs = Message Disposition Notification, the message was read
+    /// and NDNs = Non delivery notification, the message could not be delivered)
+    pub async fn handle_reports(
+        &self,
+        context: &Context,
+        from_id: u32,
+        sent_timestamp: i64,
+        parts: &[Part],
+    ) {
+        for report in &self.mdn_reports {
             for original_message_id in
                 std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
             {
                 if let Some((chat_id, msg_id)) =
-                    message::mdn_from_ext(context, from_id, original_message_id, sent_timestamp)
-                        .await
+                    message::handle_mdn(context, from_id, original_message_id, sent_timestamp).await
                 {
-                    context.emit_event(Event::MsgRead { chat_id, msg_id });
+                    context.emit_event(EventType::MsgRead { chat_id, msg_id });
                 }
             }
         }
+
+        if let Some(failure_report) = &self.failure_report {
+            let error = parts
+                .iter()
+                .find(|p| p.typ == Viewtype::Text)
+                .map(|p| p.msg.clone());
+            message::handle_ndn(context, failure_report, error).await
+        }
     }
 }
 
@@ -878,7 +1041,7 @@ async fn update_gossip_peerstates(
                 .iter()
                 .any(|info| info.addr == header.addr.to_lowercase())
             {
-                let mut peerstate = Peerstate::from_addr(context, &header.addr).await;
+                let mut peerstate = Peerstate::from_addr(context, &header.addr).await?;
                 if let Some(ref mut peerstate) = peerstate {
                     peerstate.apply_gossip(header, message_time);
                     peerstate.save_to_db(&context.sql, false).await?;
@@ -888,9 +1051,7 @@ async fn update_gossip_peerstates(
                     peerstate = Some(p);
                 }
                 if let Some(peerstate) = peerstate {
-                    if peerstate.degrade_event.is_some() {
-                        handle_degrade_event(context, &peerstate).await?;
-                    }
+                    peerstate.handle_fingerprint_change(context).await?;
                 }
 
                 gossipped_addr.insert(header.addr.clone());
@@ -914,6 +1075,13 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
+#[derive(Debug)]
+pub(crate) struct FailureReport {
+    pub rfc724_mid: String,
+    pub failed_recipient: Option<String>,
+}
+
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
     // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
     let mut msgids = Vec::new();
@@ -957,6 +1125,7 @@ pub struct Part {
     pub bytes: usize,
     pub param: Params,
     org_filename: Option<String>,
+    pub error: String,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1119,7 +1288,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_mimeparser_crash() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1131,7 +1300,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_rfc724_mid_exists() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/mail_with_message_id.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1145,7 +1314,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_rfc724_mid_not_exists() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1203,7 +1372,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_parse_first_addr() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello@one.org, world@two.org\n\
                     Chat-Disposition-Notification-To: wrong\n\
                     Content-Type: text/plain\n\
@@ -1224,7 +1393,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_mimeparser_with_context() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"From: hello\n\
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
@@ -1274,7 +1443,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_mimeparser_with_avatars() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
@@ -1315,9 +1484,31 @@ mod tests {
         assert!(mimeparser.group_avatar.unwrap().is_change());
     }
 
+    #[async_std::test]
+    async fn test_mimeparser_with_videochat() {
+        let t = TestContext::new().await;
+
+        let raw = include_bytes!("../test-data/message/videochat_invitation.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        assert_eq!(mimeparser.parts.len(), 1);
+        assert_eq!(mimeparser.parts[0].typ, Viewtype::VideochatInvitation);
+        assert_eq!(
+            mimeparser.parts[0]
+                .param
+                .get(Param::WebrtcRoom)
+                .unwrap_or_default(),
+            "https://example.org/p2p/?roomname=6HiduoAn4xN"
+        );
+        assert!(mimeparser.parts[0]
+            .msg
+            .contains("https://example.org/p2p/?roomname=6HiduoAn4xN"));
+        assert_eq!(mimeparser.user_avatar, None);
+        assert_eq!(mimeparser.group_avatar, None);
+    }
+
     #[async_std::test]
     async fn test_mimeparser_message_kml() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Chat-Version: 1.0\n\
 From: foo <foo@example.org>\n\
 To: bar <bar@example.org>\n\
@@ -1362,7 +1553,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\
 
     #[async_std::test]
     async fn test_parse_mdn() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1403,7 +1594,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
+        assert_eq!(message.mdn_reports.len(), 1);
     }
 
     /// Test parsing multiple MDNs combined in a single message.
@@ -1412,7 +1603,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
     /// multipart MIME messages.
     #[async_std::test]
     async fn test_parse_multiple_mdns() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1483,12 +1674,12 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
         );
 
         assert_eq!(message.parts.len(), 2);
-        assert_eq!(message.reports.len(), 2);
+        assert_eq!(message.mdn_reports.len(), 2);
     }
 
     #[async_std::test]
     async fn test_parse_mdn_with_additional_message_ids() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -1530,17 +1721,20 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
         );
 
         assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.reports.len(), 1);
-        assert_eq!(message.reports[0].original_message_id, "foo@example.org");
+        assert_eq!(message.mdn_reports.len(), 1);
         assert_eq!(
-            &message.reports[0].additional_message_ids,
+            message.mdn_reports[0].original_message_id,
+            "foo@example.org"
+        );
+        assert_eq!(
+            &message.mdn_reports[0].additional_message_ids,
             &["foo@example.com", "foo@example.net"]
         );
     }
 
     #[async_std::test]
     async fn test_parse_inline_attachment() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
 From: sender@example.com
 To: receiver@example.com
@@ -1580,7 +1774,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 
     #[async_std::test]
     async fn parse_inline_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"Message-ID: <foobar@example.org>
 From: foo <foo@example.org>
 Subject: example
@@ -1626,7 +1820,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
     #[async_std::test]
     async fn parse_thunderbird_html_embedded_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br#"To: Alice <alice@example.org>
 From: Bob <bob@example.org>
 Subject: Test subject
@@ -1699,7 +1893,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
     // Outlook specifies filename in the "name" attribute of Content-Type
     #[async_std::test]
     async fn parse_outlook_html_embedded_image() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let raw = br##"From: Anonymous <anonymous@example.org>
 To: Anonymous <anonymous@example.org>
 Subject: Delta Chat is great stuff!

src/oauth2.rs

@@ -1,12 +1,16 @@
 //! OAuth 2 module
 
+use regex::Regex;
 use std::collections::HashMap;
 
+use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::provider;
+use crate::provider::Oauth2Authorizer;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
@@ -15,6 +19,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
+    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -24,8 +29,11 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
+    mx_pattern: None,
 };
 
+const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -33,6 +41,7 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
+    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -53,7 +62,7 @@ pub async fn dc_get_oauth2_url(
     addr: impl AsRef<str>,
     redirect_uri: impl AsRef<str>,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
             .set_raw_config(
@@ -81,7 +90,7 @@ pub async fn dc_get_oauth2_access_token(
     code: impl AsRef<str>,
     regenerate: bool,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -239,7 +248,7 @@ pub async fn dc_get_oauth2_addr(
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    let oauth2 = Oauth2::from_address(addr.as_ref()).await?;
     oauth2.get_userinfo?;
 
     if let Some(access_token) =
@@ -263,23 +272,56 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    fn from_address(addr: impl AsRef<str>) -> Option<Self> {
+    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
         let addr_normalized = normalize_addr(addr.as_ref());
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            match domain {
-                "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
-                | "narod.ru" => Some(OAUTH2_YANDEX),
-                _ => None,
+            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
+                match &provider.oauth2_authorizer {
+                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
+                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
+                    None => None, // provider known to not support oauth2, no mx-lookup required
+                }
+            } else {
+                Oauth2::lookup_mx(domain).await
             }
         } else {
             None
         }
     }
 
+    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
+        if let Ok(resolver) = resolver(
+            config::ResolverConfig::default(),
+            config::ResolverOpts::default(),
+        )
+        .await
+        {
+            for provider in OAUTH2_PROVIDERS.iter() {
+                if let Some(pattern) = provider.mx_pattern {
+                    let re = Regex::new(pattern).unwrap();
+
+                    let mut fqdn: String = String::from(domain.as_ref());
+                    if !fqdn.ends_with('.') {
+                        fqdn.push_str(".");
+                    }
+
+                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
+                        for rr in res.iter() {
+                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
+                                return Some(provider.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
         let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
@@ -362,25 +404,39 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_oauth_from_address() {
-        assert_eq!(Oauth2::from_address("hello@gmail.com"), Some(OAUTH2_GMAIL));
+    #[async_std::test]
+    async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com"),
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com"),
+            Oauth2::from_address("hello@googlemail.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.com").await,
+            Some(OAUTH2_YANDEX)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address("hello@yandex.ru"), Some(OAUTH2_YANDEX));
 
-        assert_eq!(Oauth2::from_address("hello@web.de"), None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
+    }
+
+    #[async_std::test]
+    async fn test_oauth_from_mx() {
+        assert_eq!(
+            Oauth2::from_address("hello@google.com").await,
+            Some(OAUTH2_GMAIL)
+        );
     }
 
     #[async_std::test]
     async fn test_dc_get_oauth2_addr() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_addr(&ctx.ctx, addr, code).await;
@@ -390,7 +446,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_url() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
         let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
@@ -400,7 +456,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_token() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false).await;

src/param.rs

@@ -66,10 +66,10 @@ pub enum Param {
     Arg4 = b'H',
 
     /// For Messages
-    Error = b'L',
+    AttachGroupImage = b'A',
 
     /// For Messages
-    AttachGroupImage = b'A',
+    WebrtcRoom = b'V',
 
     /// For Messages: space-separated list of messaged IDs of forwarded copies.
     ///
@@ -174,7 +174,7 @@ impl str::FromStr for Params {
             let key = key.unwrap_or_default().trim();
             let value = value.unwrap_or_default().trim();
 
-            if let Some(key) = Param::from_u8(key.as_bytes()[0]) {
+            if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
                 inner.insert(key, value.to_string());
             } else {
                 bail!("Unknown key: {}", key);
@@ -417,7 +417,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_params_file_fs_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
@@ -427,7 +427,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_params_file_blob() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
@@ -438,7 +438,7 @@ mod tests {
     // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
     #[async_std::test]
     async fn test_params_get_fileparam() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let fname = t.dir.path().join("foo");
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());

src/peerstate.rs

@@ -1,14 +1,18 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
 use std::collections::HashSet;
-use std::convert::TryFrom;
 use std::fmt;
 
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
+use crate::chat;
+use crate::constants::Blocked;
 use crate::context::Context;
+use crate::error::{bail, Result};
+use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
+use crate::stock::StockMessage;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -39,7 +43,7 @@ pub struct Peerstate<'a> {
     pub verified_key: Option<SignedPublicKey>,
     pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
-    pub degrade_event: Option<DegradeEvent>,
+    pub fingerprint_changed: bool,
 }
 
 impl<'a> PartialEq for Peerstate<'a> {
@@ -56,7 +60,7 @@ impl<'a> PartialEq for Peerstate<'a> {
             && self.verified_key == other.verified_key
             && self.verified_key_fingerprint == other.verified_key_fingerprint
             && self.to_save == other.to_save
-            && self.degrade_event == other.degrade_event
+            && self.fingerprint_changed == other.fingerprint_changed
     }
 }
 
@@ -77,7 +81,7 @@ impl<'a> fmt::Debug for Peerstate<'a> {
             .field("verified_key", &self.verified_key)
             .field("verified_key_fingerprint", &self.verified_key_fingerprint)
             .field("to_save", &self.to_save)
-            .field("degrade_event", &self.degrade_event)
+            .field("fingerprint_changed", &self.fingerprint_changed)
             .finish()
     }
 }
@@ -89,16 +93,6 @@ pub enum ToSave {
     All = 0x02,
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
-#[repr(u8)]
-pub enum DegradeEvent {
-    /// Recoverable by an incoming encrypted mail.
-    EncryptionPaused = 0x01,
-
-    /// Recoverable by a new verify.
-    FingerprintChanged = 0x02,
-}
-
 impl<'a> Peerstate<'a> {
     pub fn new(context: &'a Context, addr: String) -> Self {
         Peerstate {
@@ -115,7 +109,7 @@ impl<'a> Peerstate<'a> {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: None,
-            degrade_event: None,
+            fingerprint_changed: false,
         }
     }
 
@@ -143,8 +137,12 @@ impl<'a> Peerstate<'a> {
         res
     }
 
-    pub async fn from_addr(context: &'a Context, addr: &str) -> Option<Peerstate<'a>> {
-        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, verified_key, verified_key_fingerprint FROM acpeerstates  WHERE addr=? COLLATE NOCASE;";
+    pub async fn from_addr(context: &'a Context, addr: &str) -> Result<Option<Peerstate<'a>>> {
+        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
+                     gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
+                     verified_key, verified_key_fingerprint \
+                     FROM acpeerstates \
+                     WHERE addr=? COLLATE NOCASE;";
         Self::from_stmt(context, query, paramsv![addr]).await
     }
 
@@ -152,7 +150,7 @@ impl<'a> Peerstate<'a> {
         context: &'a Context,
         _sql: &Sql,
         fingerprint: &Fingerprint,
-    ) -> Option<Peerstate<'a>> {
+    ) -> Result<Option<Peerstate<'a>>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
@@ -168,10 +166,10 @@ impl<'a> Peerstate<'a> {
         context: &'a Context,
         query: &str,
         params: Vec<&dyn crate::ToSql>,
-    ) -> Option<Peerstate<'a>> {
-        context
+    ) -> Result<Option<Peerstate<'a>>> {
+        let peerstate = context
             .sql
-            .query_row(query, params, |row| {
+            .query_row_optional(query, params, |row| {
                 /* all the above queries start with this: SELECT
                 addr, last_seen, last_seen_autocrypt, prefer_encrypted,
                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
@@ -187,15 +185,18 @@ impl<'a> Peerstate<'a> {
                 res.public_key_fingerprint = row
                     .get::<_, Option<String>>(7)?
                     .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
+                    .transpose()
+                    .unwrap_or_default();
                 res.gossip_key_fingerprint = row
                     .get::<_, Option<String>>(8)?
                     .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
+                    .transpose()
+                    .unwrap_or_default();
                 res.verified_key_fingerprint = row
                     .get::<_, Option<String>>(10)?
                     .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
+                    .transpose()
+                    .unwrap_or_default();
                 res.public_key = row
                     .get(4)
                     .ok()
@@ -211,8 +212,8 @@ impl<'a> Peerstate<'a> {
 
                 Ok(res)
             })
-            .await
-            .ok()
+            .await?;
+        Ok(peerstate)
     }
 
     pub fn recalc_fingerprint(&mut self) {
@@ -226,7 +227,7 @@ impl<'a> Peerstate<'a> {
             {
                 self.to_save = Some(ToSave::All);
                 if old_public_fingerprint.is_some() {
-                    self.degrade_event = Some(DegradeEvent::FingerprintChanged);
+                    self.fingerprint_changed = true;
                 }
             }
         }
@@ -240,23 +241,51 @@ impl<'a> Peerstate<'a> {
                 || old_gossip_fingerprint != self.gossip_key_fingerprint
             {
                 self.to_save = Some(ToSave::All);
-                if old_gossip_fingerprint.is_some() {
-                    self.degrade_event = Some(DegradeEvent::FingerprintChanged);
+
+                // Warn about gossip key change only if there is no public key obtained from
+                // Autocrypt header, which overrides gossip key.
+                if old_gossip_fingerprint.is_some() && self.public_key_fingerprint.is_none() {
+                    self.fingerprint_changed = true;
                 }
             }
         }
     }
 
     pub fn degrade_encryption(&mut self, message_time: i64) {
-        if self.prefer_encrypt == EncryptPreference::Mutual {
-            self.degrade_event = Some(DegradeEvent::EncryptionPaused);
-        }
-
         self.prefer_encrypt = EncryptPreference::Reset;
         self.last_seen = message_time;
         self.to_save = Some(ToSave::All);
     }
 
+    /// Adds a warning to the chat corresponding to peerstate if fingerprint has changed.
+    pub(crate) async fn handle_fingerprint_change(&self, context: &Context) -> Result<()> {
+        if self.fingerprint_changed {
+            if let Some(contact_id) = context
+                .sql
+                .query_get_value_result(
+                    "SELECT id FROM contacts WHERE addr=?;",
+                    paramsv![self.addr],
+                )
+                .await?
+            {
+                let (contact_chat_id, _) =
+                    chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Deaddrop)
+                        .await
+                        .unwrap_or_default();
+
+                let msg = context
+                    .stock_string_repl_str(StockMessage::ContactSetupChanged, self.addr.clone())
+                    .await;
+
+                chat::add_info_msg(context, contact_chat_id, msg).await;
+                emit_event!(context, EventType::ChatModified(contact_chat_id));
+            } else {
+                bail!("contact with peerstate.addr {:?} not found", &self.addr);
+            }
+        }
+        Ok(())
+    }
+
     pub fn apply_header(&mut self, header: &Aheader, message_time: i64) {
         if self.addr.to_lowercase() != header.addr.to_lowercase() {
             return;
@@ -270,11 +299,6 @@ impl<'a> Peerstate<'a> {
                 || header.prefer_encrypt == EncryptPreference::NoPreference)
                 && header.prefer_encrypt != self.prefer_encrypt
             {
-                if self.prefer_encrypt == EncryptPreference::Mutual
-                    && header.prefer_encrypt != EncryptPreference::Mutual
-                {
-                    self.degrade_event = Some(DegradeEvent::EncryptionPaused);
-                }
                 self.prefer_encrypt = header.prefer_encrypt;
                 self.to_save = Some(ToSave::All)
             }
@@ -324,11 +348,9 @@ impl<'a> Peerstate<'a> {
 
     pub fn render_gossip_header(&self, min_verified: PeerstateVerifiedStatus) -> Option<String> {
         if let Some(key) = self.peek_key(min_verified) {
-            // TODO: avoid cloning
-            let public_key = SignedPublicKey::try_from(key.clone()).ok()?;
             let header = Aheader::new(
                 self.addr.clone(),
-                public_key,
+                key.clone(), // TODO: avoid cloning
                 // Autocrypt 1.1.0 specification says that
                 // `prefer-encrypt` attribute SHOULD NOT be included,
                 // but we include it anyway to propagate encryption
@@ -403,21 +425,20 @@ impl<'a> Peerstate<'a> {
     }
 
     pub async fn save_to_db(&self, sql: &Sql, create: bool) -> crate::sql::Result<()> {
-        if create {
-            sql.execute(
-                "INSERT INTO acpeerstates (addr) VALUES(?);",
-                paramsv![self.addr],
-            )
-            .await?;
-        }
-
         if self.to_save == Some(ToSave::All) || create {
             sql.execute(
+                if create {
+                "INSERT INTO acpeerstates (last_seen, last_seen_autocrypt, prefer_encrypted, \
+                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
+                 verified_key, verified_key_fingerprint, addr \
+                ) VALUES(?,?,?,?,?,?,?,?,?,?,?)"
+                } else {
                 "UPDATE acpeerstates \
                  SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
                  public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
                  verified_key=?, verified_key_fingerprint=? \
-                 WHERE addr=?;",
+                 WHERE addr=?"
+                },
                 paramsv![
                     self.last_seen,
                     self.last_seen_autocrypt,
@@ -450,14 +471,11 @@ impl<'a> Peerstate<'a> {
     }
 
     pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
-        if self.verified_key.is_some() && self.verified_key_fingerprint.is_some() {
-            let vkc = self.verified_key_fingerprint.as_ref().unwrap();
-            if fingerprints.contains(vkc) {
-                return true;
-            }
+        if let Some(vkc) = &self.verified_key_fingerprint {
+            fingerprints.contains(vkc) && self.verified_key.is_some()
+        } else {
+            false
         }
-
-        false
     }
 }
 
@@ -476,7 +494,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
         let pub_key = alice_keypair().public;
@@ -495,7 +513,7 @@ mod tests {
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -505,7 +523,8 @@ mod tests {
 
         let peerstate_new = Peerstate::from_addr(&ctx.ctx, addr)
             .await
-            .expect("failed to load peerstate from db");
+            .expect("failed to load peerstate from db")
+            .expect("no peerstate found in the database");
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
@@ -513,13 +532,14 @@ mod tests {
         let peerstate_new2 =
             Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint())
                 .await
-                .expect("failed to load peerstate from db");
+                .expect("failed to load peerstate from db")
+                .expect("no peerstate found in the database");
         assert_eq!(peerstate, peerstate_new2);
     }
 
     #[async_std::test]
     async fn test_peerstate_double_create() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
         let pub_key = alice_keypair().public;
 
@@ -537,7 +557,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -552,7 +572,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_with_empty_gossip_key_save_to_db() {
-        let ctx = crate::test_utils::dummy_context().await;
+        let ctx = crate::test_utils::TestContext::new().await;
         let addr = "hello@mail.com";
 
         let pub_key = alice_keypair().public;
@@ -571,7 +591,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -585,7 +605,36 @@ mod tests {
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
-        assert_eq!(peerstate, peerstate_new);
+        assert_eq!(Some(peerstate), peerstate_new);
+    }
+
+    #[async_std::test]
+    async fn test_peerstate_load_db_defaults() {
+        let ctx = crate::test_utils::TestContext::new().await;
+        let addr = "hello@mail.com";
+
+        // Old code created peerstates with this code and updated
+        // other values later.  If UPDATE failed, other columns had
+        // default values, in particular fingerprints were set to
+        // empty strings instead of NULL. This should not be the case
+        // anymore, but the regression test still checks that defaults
+        // can be loaded without errors.
+        ctx.ctx
+            .sql
+            .execute("INSERT INTO acpeerstates (addr) VALUES(?)", paramsv![addr])
+            .await
+            .expect("Failed to write to the database");
+
+        let peerstate = Peerstate::from_addr(&ctx.ctx, addr)
+            .await
+            .expect("Failed to load peerstate from db")
+            .expect("Loaded peerstate is empty");
+
+        // Check that default values for fingerprints are treated like
+        // NULL.
+        assert_eq!(peerstate.public_key_fingerprint, None);
+        assert_eq!(peerstate.gossip_key_fingerprint, None);
+        assert_eq!(peerstate.verified_key_fingerprint, None);
     }
 
     // TODO: don't copy this from stress.rs

src/pgp.rs

@@ -272,6 +272,14 @@ pub async fn pk_encrypt(
     .await
 }
 
+/// Decrypts the message with keys from the private key keyring.
+///
+/// Receiver private keys are provided in
+/// `private_keys_for_decryption`.
+///
+/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// of all keys from the `public_keys_for_validation` keyring that
+/// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
 pub async fn pk_decrypt(
     ctext: Vec<u8>,
@@ -290,36 +298,41 @@ pub async fn pk_decrypt(
     })
     .await?;
 
-    ensure!(!msgs.is_empty(), "No valid messages found");
+    if let Some(msg) = msgs.into_iter().next() {
+        // get_content() will decompress the message if needed,
+        // but this avoids decompressing it again to check signatures
+        let msg = msg.decompress()?;
 
-    let content = match msgs[0].get_content()? {
-        Some(content) => content,
-        None => bail!("Decrypted message is empty"),
-    };
+        let content = match msg.get_content()? {
+            Some(content) => content,
+            None => bail!("The decrypted message is empty"),
+        };
 
-    if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-        if !public_keys_for_validation.is_empty() {
-            let fingerprints = async_std::task::spawn_blocking(move || {
-                let dec_msg = &msgs[0];
+        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
+            if !public_keys_for_validation.is_empty() {
+                let fingerprints = async_std::task::spawn_blocking(move || {
+                    let pkeys = public_keys_for_validation.keys();
 
-                let pkeys = public_keys_for_validation.keys();
-
-                let mut fingerprints: Vec<Fingerprint> = Vec::new();
-                for pkey in pkeys {
-                    if dec_msg.verify(&pkey.primary_key).is_ok() {
-                        let fp = DcKey::fingerprint(pkey);
-                        fingerprints.push(fp);
+                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                        for pkey in pkeys {
+                            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                                let fp = DcKey::fingerprint(pkey);
+                                fingerprints.push(fp);
+                            }
+                        }
                     }
-                }
-                fingerprints
-            })
-            .await;
+                    fingerprints
+                })
+                .await;
 
-            ret_signature_fingerprints.extend(fingerprints);
+                ret_signature_fingerprints.extend(fingerprints);
+            }
         }
+        Ok(content)
+    } else {
+        bail!("No valid messages found");
     }
-
-    Ok(content)
 }
 
 /// Symmetric encryption.
@@ -352,11 +365,13 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
         let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
         let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-        ensure!(!msgs.is_empty(), "No valid messages found");
-
-        match msgs[0].get_content()? {
-            Some(content) => Ok(content),
-            None => bail!("Decrypted message is empty"),
+        if let Some(msg) = msgs.first() {
+            match msg.get_content()? {
+                Some(content) => Ok(content),
+                None => bail!("Decrypted message is empty"),
+            }
+        } else {
+            bail!("No valid messages found")
         }
     })
     .await

src/provider/data.rs

@@ -20,6 +20,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // aol.md: aol.com
@@ -32,6 +33,22 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // arcor.de.md: arcor.de
+    static ref P_ARCOR_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/arcor-de",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.arcor.de", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "mail.arcor.de", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // autistici.org.md: autistici.org
@@ -46,6 +63,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -60,6 +78,22 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // buzon.uy.md: buzon.uy
+    static ref P_BUZON_UY: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/buzon-uy",
+        server: vec![
+            Server { protocol: IMAP, socket: STARTTLS, hostname: "buzon.uy", port: 143, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "buzon.uy", port: 587, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // chello.at.md: chello.at
@@ -74,13 +108,34 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_COMCAST: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/comcast",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // dismail.de.md: dismail.de
-    // - skipping provider with status OK and no special things to do
+    static ref P_DISMAIL_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dismail-de",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // disroot.md: disroot.org
     static ref P_DISROOT: Provider = Provider {
@@ -92,6 +147,28 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: true,
+        oauth2_authorizer: None,
+    };
+
+    // dubby.org.md: dubby.org
+    static ref P_DUBBY_ORG: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/dubby-org",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "dubby.org", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "dubby.org", port: 587, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "dubby.org", port: 465, username_pattern: EMAIL },
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
+        strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // example.com.md: example.com, example.org
@@ -106,6 +183,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // fastmail.md: fastmail.com
@@ -118,6 +196,20 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // firemail.de.md: firemail.at, firemail.de
+    static ref P_FIREMAIL_DE: Provider = Provider {
+        status: Status::PREPARATION,
+        before_login_hint: "Firemail erlaubt nur bei bezahlten Accounts den vollen Zugriff auf das E-Mail-Protokoll. Wenn Sie nicht für Firemail bezahlen, verwenden Sie bitte einen anderen E-Mail-Anbieter.",
+        after_login_hint: "Leider schränkt Firemail die maximale Gruppengröße ein. Je nach Bezahlmodell sind nur 5 bis 30 Gruppenmitglieder erlaubt.",
+        overview_page: "https://providers.delta.chat/firemail-de",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // five.chat.md: five.chat
@@ -127,9 +219,18 @@ lazy_static::lazy_static! {
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/five-chat",
         server: vec![
+            Server { protocol: IMAP, socket: STARTTLS, hostname: "five.chat", port: 143, username_pattern: EMAIL },
+            Server { protocol: IMAP, socket: SSL, hostname: "five.chat", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "five.chat", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
         strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // freenet.de.md: freenet.de
@@ -144,6 +245,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -158,6 +260,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -173,10 +276,34 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // hey.com.md: hey.com
+    static ref P_HEY_COM: Provider = Provider {
+        status: Status::BROKEN,
+        before_login_hint: "hey.com does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to hey.com.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/hey-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // i.ua.md: i.ua
-    // - skipping provider with status OK and no special things to do
+    static ref P_I_UA: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/i-ua",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // icloud.md: icloud.com, me.com, mac.com
     static ref P_ICLOUD: Provider = Provider {
@@ -190,16 +317,47 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // kolst.com.md: kolst.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KOLST_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kolst-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // kontent.com.md: kontent.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_KONTENT_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/kontent-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
-    // - skipping provider with status OK and no special things to do
+    static ref P_MAIL_RU: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/mail-ru",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
     static ref P_MAILBOX_ORG: Provider = Provider {
@@ -211,6 +369,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // nauta.cu.md: nauta.cu
@@ -233,6 +392,7 @@ lazy_static::lazy_static! {
             ConfigDefault { key: Config::MediaQuality, value: "1" },
         ]),
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -247,6 +407,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
@@ -261,6 +422,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -273,6 +435,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // riseup.net.md: riseup.net
@@ -285,10 +448,21 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // rogers.com.md: rogers.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_ROGERS_COM: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/rogers-com",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // systemli.org.md: systemli.org
     static ref P_SYSTEMLI_ORG: Provider = Provider {
@@ -300,6 +474,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // t-online.md: t-online.de, magenta.de
@@ -312,6 +487,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // testrun.md: testrun.org
@@ -325,8 +501,14 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::BccSelf, value: "1" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+        ]),
         strict_tls: true,
+        oauth2_authorizer: None,
     };
 
     // tiscali.it.md: tiscali.it
@@ -341,13 +523,64 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // ukr.net.md: ukr.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_UKR_NET: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/ukr-net",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // undernet.uy.md: undernet.uy
+    static ref P_UNDERNET_UY: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/undernet-uy",
+        server: vec![
+            Server { protocol: IMAP, socket: STARTTLS, hostname: "undernet.uy", port: 143, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "undernet.uy", port: 587, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
 
     // vfemail.md: vfemail.net
-    // - skipping provider with status OK and no special things to do
+    static ref P_VFEMAIL: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/vfemail",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
+    // vodafone.de.md: vodafone.de, vodafonemail.de
+    static ref P_VODAFONE_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/vodafone-de",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.vodafonemail.de", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.vodafonemail.de", port: 587, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
 
     // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
     static ref P_WEB_DE: Provider = Provider {
@@ -362,6 +595,7 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
@@ -376,18 +610,22 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
-    // yandex.ru.md: yandex.ru, yandex.com
+    // yandex.ru.md: yandex.com, yandex.by, yandex.kz, yandex.ru, yandex.ua, ya.ru, narod.ru
     static ref P_YANDEX_RU: Provider = Provider {
         status: Status::PREPARATION,
         before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yandex-ru",
         server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.yandex.com", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "smtp.yandex.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
         strict_tls: true,
+        oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -402,18 +640,27 @@ lazy_static::lazy_static! {
         ],
         config_defaults: None,
         strict_tls: false,
+        oauth2_authorizer: None,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
+        ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
+        ("xfinity.com", &*P_COMCAST),
+        ("comcast.net", &*P_COMCAST),
+        ("dismail.de", &*P_DISMAIL_DE),
         ("disroot.org", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("firemail.at", &*P_FIREMAIL_DE),
+        ("firemail.de", &*P_FIREMAIL_DE),
         ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
@@ -427,9 +674,17 @@ lazy_static::lazy_static! {
         ("gmx.info", &*P_GMX_NET),
         ("gmx.biz", &*P_GMX_NET),
         ("gmx.com", &*P_GMX_NET),
+        ("hey.com", &*P_HEY_COM),
+        ("i.ua", &*P_I_UA),
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
+        ("kolst.com", &*P_KOLST_COM),
+        ("kontent.com", &*P_KONTENT_COM),
+        ("mail.ru", &*P_MAIL_RU),
+        ("inbox.ru", &*P_MAIL_RU),
+        ("bk.ru", &*P_MAIL_RU),
+        ("list.ru", &*P_MAIL_RU),
         ("mailbox.org", &*P_MAILBOX_ORG),
         ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
@@ -490,11 +745,17 @@ lazy_static::lazy_static! {
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
         ("riseup.net", &*P_RISEUP_NET),
+        ("rogers.com", &*P_ROGERS_COM),
         ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
+        ("ukr.net", &*P_UKR_NET),
+        ("undernet.uy", &*P_UNDERNET_UY),
+        ("vfemail.net", &*P_VFEMAIL),
+        ("vodafone.de", &*P_VODAFONE_DE),
+        ("vodafonemail.de", &*P_VODAFONE_DE),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),
         ("flirt.ms", &*P_WEB_DE),
@@ -537,8 +798,13 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
-        ("yandex.ru", &*P_YANDEX_RU),
         ("yandex.com", &*P_YANDEX_RU),
+        ("yandex.by", &*P_YANDEX_RU),
+        ("yandex.kz", &*P_YANDEX_RU),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("yandex.ua", &*P_YANDEX_RU),
+        ("ya.ru", &*P_YANDEX_RU),
+        ("narod.ru", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/provider/mod.rs

@@ -35,6 +35,13 @@ pub enum UsernamePattern {
     EMAILLOCALPART = 2,
 }
 
+#[derive(Debug, PartialEq)]
+#[repr(u8)]
+pub enum Oauth2Authorizer {
+    Yandex = 1,
+    Gmail = 2,
+}
+
 #[derive(Debug)]
 pub struct Server {
     pub protocol: Protocol,
@@ -73,6 +80,7 @@ pub struct Provider {
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
     pub strict_tls: bool,
+    pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
 impl Provider {

src/provider/update.py

@@ -103,6 +103,9 @@ def process_data(data, file):
     strict_tls = data.get("strict_tls", False)
     strict_tls = "true" if strict_tls else "false"
 
+    oauth2 = data.get("oauth2", "")
+    oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -115,6 +118,7 @@ def process_data(data, file):
         provider += "        server: vec![\n" + server + "        ],\n"
         provider += "        config_defaults: " + config_defaults + ",\n"
         provider += "        strict_tls: " + strict_tls + ",\n"
+        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -125,11 +129,11 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None" and strict_tls == "false":
-        out_all += "    // - skipping provider with status OK and no special things to do\n\n"
-    else:
-        out_all += provider
-        out_domains += domains
+
+    # also add provider with no special things to do -
+    # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
+    out_all += provider
+    out_domains += domains
 
 
 def process_file(file):

src/qr.rs

@@ -12,11 +12,13 @@ use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
 use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
+use crate::message::Message;
 use crate::param::*;
 use crate::peerstate::*;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
+const DCWEBRTC_SCHEME: &str = "DCWEBRTC:";
 const MAILTO_SCHEME: &str = "mailto:";
 const MATMSG_SCHEME: &str = "MATMSG:";
 const VCARD_SCHEME: &str = "BEGIN:VCARD";
@@ -51,6 +53,8 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
         decode_openpgp(context, qr).await
     } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(context, qr)
+    } else if starts_with_ignore_case(qr, DCWEBRTC_SCHEME) {
+        decode_webrtc_instance(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
         decode_mailto(context, qr).await
     } else if qr.starts_with(SMTP_SCHEME) {
@@ -68,6 +72,7 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
 /// scheme: `OPENPGP4FPR:FINGERPRINT#a=ADDR&n=NAME&i=INVITENUMBER&s=AUTH`
 ///     or: `OPENPGP4FPR:FINGERPRINT#a=ADDR&g=GROUPNAME&x=GROUPID&i=INVITENUMBER&s=AUTH`
+#[allow(clippy::indexing_slicing)]
 async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
@@ -138,7 +143,10 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
-    let peerstate = Peerstate::from_fingerprint(context, &context.sql, &fingerprint).await;
+    let peerstate = match Peerstate::from_fingerprint(context, &context.sql, &fingerprint).await {
+        Ok(peerstate) => peerstate,
+        Err(err) => return format_err!("Can't load peerstate: {}", err).into(),
+    };
 
     if invitenumber.is_none() || auth.is_none() {
         if let Some(peerstate) = peerstate {
@@ -187,13 +195,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 }
 
 /// scheme: `DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3`
+#[allow(clippy::indexing_slicing)]
 fn decode_account(_context: &Context, qr: &str) -> Lot {
     let payload = &qr[DCACCOUNT_SCHEME.len()..];
 
     let mut lot = Lot::new();
 
     if let Ok(url) = url::Url::parse(payload) {
-        if url.scheme() == "https" {
+        if url.scheme() == "http" || url.scheme() == "https" {
             lot.state = LotState::QrAccount;
             lot.text1 = url.host_str().map(|x| x.to_string());
         } else {
@@ -208,6 +217,31 @@ fn decode_account(_context: &Context, qr: &str) -> Lot {
     lot
 }
 
+/// scheme: `DCWEBRTC:https://meet.jit.si/$ROOM`
+#[allow(clippy::indexing_slicing)]
+fn decode_webrtc_instance(_context: &Context, qr: &str) -> Lot {
+    let payload = &qr[DCWEBRTC_SCHEME.len()..];
+
+    let mut lot = Lot::new();
+
+    let (_type, url) = Message::parse_webrtc_instance(payload);
+    if let Ok(url) = url::Url::parse(&url) {
+        if url.scheme() == "http" || url.scheme() == "https" {
+            lot.state = LotState::QrWebrtcInstance;
+            lot.text1 = url.host_str().map(|x| x.to_string());
+            lot.text2 = Some(payload.to_string())
+        } else {
+            lot.state = LotState::QrError;
+            lot.text1 = Some(format!("Bad scheme for webrtc instance: {}", payload));
+        }
+    } else {
+        lot.state = LotState::QrError;
+        lot.text1 = Some(format!("Invalid webrtc instance: {}", payload));
+    }
+
+    lot
+}
+
 #[derive(Debug, Deserialize)]
 struct CreateAccountResponse {
     email: String,
@@ -217,7 +251,8 @@ struct CreateAccountResponse {
 /// take a qr of the type DC_QR_ACCOUNT, parse it's parameters,
 /// download additional information from the contained url and set the parameters.
 /// on success, a configure::configure() should be able to log in to the account
-pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+#[allow(clippy::indexing_slicing)]
+async fn set_account_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
 
     let response: Result<CreateAccountResponse, surf::Error> =
@@ -237,9 +272,24 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error
     Ok(())
 }
 
+pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+    match check_qr(context, &qr).await.state {
+        LotState::QrAccount => set_account_from_qr(context, qr).await,
+        LotState::QrWebrtcInstance => {
+            let val = decode_webrtc_instance(context, qr).text2;
+            context
+                .set_config(Config::WebrtcInstance, val.as_deref())
+                .await?;
+            Ok(())
+        }
+        _ => bail!("qr code does not contain config: {}", qr),
+    }
+}
+
 /// Extract address for the mailto scheme.
 ///
 /// Scheme: `mailto:addr...?subject=...&body=..`
+#[allow(clippy::indexing_slicing)]
 async fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let payload = &qr[MAILTO_SCHEME.len()..];
 
@@ -261,6 +311,7 @@ async fn decode_mailto(context: &Context, qr: &str) -> Lot {
 /// Extract address for the smtp scheme.
 ///
 /// Scheme: `SMTP:addr...:subject...:body...`
+#[allow(clippy::indexing_slicing)]
 async fn decode_smtp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[SMTP_SCHEME.len()..];
 
@@ -283,6 +334,7 @@ async fn decode_smtp(context: &Context, qr: &str) -> Lot {
 /// Scheme: `MATMSG:TO:addr...;SUB:subject...;BODY:body...;`
 ///
 /// There may or may not be linebreaks after the fields.
+#[allow(clippy::indexing_slicing)]
 async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     // Does not work when the text `TO:` is used in subject/body _and_ TO: is not the first field.
     // we ignore this case.
@@ -316,14 +368,15 @@ lazy_static! {
 /// Extract address for the matmsg scheme.
 ///
 /// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
+#[allow(clippy::indexing_slicing)]
 async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
         .captures(qr)
-        .map(|caps| {
-            let last_name = &caps[1];
-            let first_name = &caps[2];
+        .and_then(|caps| {
+            let last_name = caps.get(1)?.as_str().trim();
+            let first_name = caps.get(2)?.as_str().trim();
 
-            format!("{} {}", first_name.trim(), last_name.trim())
+            Some(format!("{} {}", first_name, last_name))
         })
         .unwrap_or_default();
 
@@ -383,11 +436,11 @@ fn normalize_address(addr: &str) -> Result<String, Error> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_decode_http() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "http://www.hello.com").await;
 
@@ -399,7 +452,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_https() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "https://www.hello.com").await;
 
@@ -411,7 +464,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_text() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "I am so cool").await;
 
@@ -423,7 +476,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_vcard() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -441,7 +494,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_matmsg() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -459,7 +512,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_mailto() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -485,7 +538,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_smtp() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld").await;
 
@@ -499,7 +552,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_group() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -528,7 +581,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_secure_join() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -556,7 +609,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_openpgp_without_addr() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -591,7 +644,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_decode_account() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
 
         let res = check_qr(
             &ctx.ctx,
@@ -611,12 +664,31 @@ mod tests {
         assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
+    #[async_std::test]
+    async fn test_decode_webrtc_instance() {
+        let ctx = TestContext::new().await;
+
+        let res = check_qr(&ctx.ctx, "DCWEBRTC:basicwebrtc:https://basicurl.com/$ROOM").await;
+        assert_eq!(res.get_state(), LotState::QrWebrtcInstance);
+        assert_eq!(res.get_text1().unwrap(), "basicurl.com");
+        assert_eq!(
+            res.get_text2().unwrap(),
+            "basicwebrtc:https://basicurl.com/$ROOM"
+        );
+
+        // Test it again with mixcased "dcWebRTC:" uri scheme
+        let res = check_qr(&ctx.ctx, "dcWebRTC:https://example.org/").await;
+        assert_eq!(res.get_state(), LotState::QrWebrtcInstance);
+        assert_eq!(res.get_text1().unwrap(), "example.org");
+        assert_eq!(res.get_text2().unwrap(), "https://example.org/");
+    }
+
     #[async_std::test]
     async fn test_decode_account_bad_scheme() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
-            "DCACCOUNT:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "DCACCOUNT:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
@@ -625,10 +697,40 @@ mod tests {
         // Test it again with lowercased "dcaccount:" uri scheme
         let res = check_qr(
             &ctx.ctx,
-            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "dcaccount:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
     }
+
+    #[async_std::test]
+    async fn test_set_config_from_qr() {
+        let ctx = TestContext::new().await;
+
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "badqr:https://example.org/").await;
+        assert!(!res.is_ok());
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "https://no.qr").await;
+        assert!(!res.is_ok());
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "dcwebrtc:https://example.org/").await;
+        assert!(res.is_ok());
+        assert_eq!(
+            ctx.ctx.get_config(Config::WebrtcInstance).await.unwrap(),
+            "https://example.org/"
+        );
+
+        let res =
+            set_config_from_qr(&ctx.ctx, "DCWEBRTC:basicwebrtc:https://foo.bar/?$ROOM&test").await;
+        assert!(res.is_ok());
+        assert_eq!(
+            ctx.ctx.get_config(Config::WebrtcInstance).await.unwrap(),
+            "basicwebrtc:https://foo.bar/?$ROOM&test"
+        );
+    }
 }

src/scheduler.rs

@@ -36,14 +36,6 @@ impl Context {
         self.scheduler.read().await.interrupt_inbox(info).await;
     }
 
-    pub(crate) async fn interrupt_sentbox(&self, info: InterruptInfo) {
-        self.scheduler.read().await.interrupt_sentbox(info).await;
-    }
-
-    pub(crate) async fn interrupt_mvbox(&self, info: InterruptInfo) {
-        self.scheduler.read().await.interrupt_mvbox(info).await;
-    }
-
     pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
         self.scheduler.read().await.interrupt_smtp(info).await;
     }
@@ -82,7 +74,18 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 }
                 None => {
                     jobs_loaded = 0;
-                    info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
+
+                    // Expunge folder if needed, e.g. if some jobs have
+                    // deleted messages on the server.
+                    if let Err(err) = connection.maybe_close_folder(&ctx).await {
+                        warn!(ctx, "failed to close folder: {:?}", err);
+                    }
+
+                    info = if ctx.get_config_bool(Config::InboxWatch).await {
+                        fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
+                    } else {
+                        connection.fake_idle(&ctx, None).await
+                    };
                 }
             }
         }
@@ -102,7 +105,7 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
             if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "{}", err);
+                error_network!(ctx, "{}", err);
                 return;
             }
 
@@ -124,7 +127,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
         Some(watch_folder) => {
             // connect and fake idle if unable to connect
             if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "imap connection failed: {}", err);
+                warn!(ctx, "imap connection failed: {}", err);
                 return connection.fake_idle(&ctx, None).await;
             }
 
@@ -243,29 +246,21 @@ impl Scheduler {
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
         let (inbox, inbox_handlers) = ImapConnectionState::new();
 
-        *self = Scheduler::Running {
-            inbox,
-            mvbox,
-            sentbox,
-            smtp,
-            inbox_handle: None,
-            mvbox_handle: None,
-            sentbox_handle: None,
-            smtp_handle: None,
-        };
-
         let (inbox_start_send, inbox_start_recv) = channel(1);
-        if let Scheduler::Running { inbox_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *inbox_handle = Some(task::spawn(async move {
-                inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
-            }));
-        }
-
         let (mvbox_start_send, mvbox_start_recv) = channel(1);
-        if let Scheduler::Running { mvbox_handle, .. } = self {
+        let mut mvbox_handle = None;
+        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let mut sentbox_handle = None;
+        let (smtp_start_send, smtp_start_recv) = channel(1);
+
+        let ctx1 = ctx.clone();
+        let inbox_handle = Some(task::spawn(async move {
+            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
+        }));
+
+        if ctx.get_config_bool(Config::MvboxWatch).await {
             let ctx1 = ctx.clone();
-            *mvbox_handle = Some(task::spawn(async move {
+            mvbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
                     ctx1,
                     mvbox_start_send,
@@ -274,12 +269,13 @@ impl Scheduler {
                 )
                 .await
             }));
+        } else {
+            mvbox_start_send.send(()).await;
         }
 
-        let (sentbox_start_send, sentbox_start_recv) = channel(1);
-        if let Scheduler::Running { sentbox_handle, .. } = self {
+        if ctx.get_config_bool(Config::SentboxWatch).await {
             let ctx1 = ctx.clone();
-            *sentbox_handle = Some(task::spawn(async move {
+            sentbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
                     ctx1,
                     sentbox_start_send,
@@ -288,15 +284,25 @@ impl Scheduler {
                 )
                 .await
             }));
+        } else {
+            sentbox_start_send.send(()).await;
         }
 
-        let (smtp_start_send, smtp_start_recv) = channel(1);
-        if let Scheduler::Running { smtp_handle, .. } = self {
-            let ctx1 = ctx.clone();
-            *smtp_handle = Some(task::spawn(async move {
-                smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
-            }));
-        }
+        let ctx1 = ctx.clone();
+        let smtp_handle = Some(task::spawn(async move {
+            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
+        }));
+
+        *self = Scheduler::Running {
+            inbox,
+            mvbox,
+            sentbox,
+            smtp,
+            inbox_handle,
+            mvbox_handle,
+            sentbox_handle,
+            smtp_handle,
+        };
 
         // wait for all loops to be started
         if let Err(err) = inbox_start_recv
@@ -386,10 +392,18 @@ impl Scheduler {
                 smtp_handle,
                 ..
             } => {
-                inbox_handle.take().expect("inbox not started").await;
-                mvbox_handle.take().expect("mvbox not started").await;
-                sentbox_handle.take().expect("sentbox not started").await;
-                smtp_handle.take().expect("smtp not started").await;
+                if let Some(handle) = inbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = mvbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = sentbox_handle.take() {
+                    handle.await;
+                }
+                if let Some(handle) = smtp_handle.take() {
+                    handle.await;
+                }
 
                 *self = Scheduler::Stopped;
             }

src/securejoin.rs

@@ -12,7 +12,7 @@ use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
 use crate::error::{bail, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::lot::LotState;
@@ -32,7 +32,7 @@ macro_rules! joiner_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::SecurejoinJoinerProgress {
+        $context.emit_event($crate::events::EventType::SecurejoinJoinerProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -45,7 +45,7 @@ macro_rules! inviter_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::SecurejoinInviterProgress {
+        $context.emit_event($crate::events::EventType::SecurejoinInviterProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -114,7 +114,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
 
             Some(format!(
                 "OPENPGP4FPR:{}#a={}&g={}&x={}&i={}&s={}",
-                fingerprint,
+                fingerprint.hex(),
                 self_addr_urlencoded,
                 &group_name_urlencoded,
                 &chat.grpid,
@@ -129,7 +129,11 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
         // parameters used: a=n=i=s=
         Some(format!(
             "OPENPGP4FPR:{}#a={}&n={}&i={}&s={}",
-            fingerprint, self_addr_urlencoded, self_name_urlencoded, &invitenumber, &auth,
+            fingerprint.hex(),
+            self_addr_urlencoded,
+            self_name_urlencoded,
+            &invitenumber,
+            &auth,
         ))
     };
 
@@ -348,9 +352,8 @@ async fn send_handshake_msg(
 }
 
 async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32 {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
-    if contacts.len() == 1 {
-        contacts[0]
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        contact_id
     } else {
         0
     }
@@ -361,11 +364,22 @@ async fn fingerprint_equals_sender(
     fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
 ) -> bool {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+            let peerstate = match Peerstate::from_addr(context, contact.get_addr()).await {
+                Ok(peerstate) => peerstate,
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Failed to sender peerstate for {}: {}",
+                        contact.get_addr(),
+                        err
+                    );
+                    return false;
+                }
+            };
 
-    if contacts.len() == 1 {
-        if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
-            if let Some(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
+            if let Some(peerstate) = peerstate {
                 if peerstate.public_key_fingerprint.is_some()
                     && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
@@ -422,6 +436,7 @@ pub(crate) enum HandshakeMessage {
 /// When handle_securejoin_handshake() is called,
 /// the message is not yet filed in the database;
 /// this is done by receive_imf() later on as needed.
+#[allow(clippy::indexing_slicing)]
 pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
     mime_message: &MimeMessage,
@@ -638,7 +653,7 @@ pub(crate) async fn handle_securejoin_handshake(
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await;
             info!(context, "Auth verified.",);
             secure_connection_established(context, contact_chat_id).await;
-            emit_event!(context, Event::ContactsChanged(Some(contact_id)));
+            emit_event!(context, EventType::ContactsChanged(Some(contact_id)));
             inviter_progress!(context, contact_id, 600);
             if join_vg {
                 // the vg-member-added message is special:
@@ -758,7 +773,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 return Ok(abort_retval);
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined).await;
-            emit_event!(context, Event::ContactsChanged(None));
+            emit_event!(context, EventType::ContactsChanged(None));
             let cg_member_added = mime_message
                 .get(HeaderDef::ChatGroupMemberAdded)
                 .map(|s| s.as_str())
@@ -945,7 +960,7 @@ async fn secure_connection_established(context: &Context, contact_chat_id: ChatI
         .stock_string_repl_str(StockMessage::ContactVerified, addr)
         .await;
     chat::add_info_msg(context, contact_chat_id, msg).await;
-    emit_event!(context, Event::ChatModified(contact_chat_id));
+    emit_event!(context, EventType::ChatModified(contact_chat_id));
 }
 
 async fn could_not_establish_secure_connection(
@@ -972,7 +987,7 @@ async fn could_not_establish_secure_connection(
 
 async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
     if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await
+        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await?
     {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
@@ -1006,65 +1021,18 @@ fn encrypted_and_signed(
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
         false
-    } else if mimeparser.signatures.is_empty() {
-        warn!(context, "Message not signed.",);
-        false
-    } else if expected_fingerprint.is_none() {
+    } else if let Some(expected_fingerprint) = expected_fingerprint {
+        if !mimeparser.signatures.contains(expected_fingerprint) {
+            warn!(
+                context,
+                "Message does not match expected fingerprint {}.", expected_fingerprint,
+            );
+            false
+        } else {
+            true
+        }
+    } else {
         warn!(context, "Fingerprint for comparison missing.");
         false
-    } else if !mimeparser
-        .signatures
-        .contains(expected_fingerprint.unwrap())
-    {
-        warn!(
-            context,
-            "Message does not match expected fingerprint {}.",
-            expected_fingerprint.unwrap(),
-        );
-        false
-    } else {
-        true
     }
 }
-
-pub async fn handle_degrade_event(
-    context: &Context,
-    peerstate: &Peerstate<'_>,
-) -> Result<(), Error> {
-    // - we do not issue an warning for DC_DE_ENCRYPTION_PAUSED as this is quite normal
-    // - currently, we do not issue an extra warning for DC_DE_VERIFICATION_LOST - this always comes
-    //   together with DC_DE_FINGERPRINT_CHANGED which is logged, the idea is not to bother
-    //   with things they cannot fix, so the user is just kicked from the verified group
-    //   (and he will know this and can fix this)
-    if Some(DegradeEvent::FingerprintChanged) == peerstate.degrade_event {
-        let contact_id: i32 = match context
-            .sql
-            .query_get_value(
-                context,
-                "SELECT id FROM contacts WHERE addr=?;",
-                paramsv![peerstate.addr],
-            )
-            .await
-        {
-            None => bail!(
-                "contact with peerstate.addr {:?} not found",
-                &peerstate.addr
-            ),
-            Some(contact_id) => contact_id,
-        };
-        if contact_id > 0 {
-            let (contact_chat_id, _) =
-                chat::create_or_lookup_by_contact_id(context, contact_id as u32, Blocked::Deaddrop)
-                    .await
-                    .unwrap_or_default();
-
-            let msg = context
-                .stock_string_repl_str(StockMessage::ContactSetupChanged, peerstate.addr.clone())
-                .await;
-
-            chat::add_info_msg(context, contact_chat_id, msg).await;
-            emit_event!(context, Event::ChatModified(contact_chat_id));
-        }
-    }
-    Ok(())
-}

src/simplify.rs

@@ -7,14 +7,15 @@
 // but for non-delta-compatibility, that seems to be better.
 // (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
 pub fn escape_message_footer_marks(text: &str) -> String {
-    if text.starts_with("--") {
-        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
+    if let Some(text) = text.strip_prefix("--") {
+        "-\u{200B}-".to_string() + &text.replace("\n--", "\n-\u{200B}-")
     } else {
         text.replace("\n--", "\n-\u{200B}-")
     }
 }
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
+#[allow(clippy::indexing_slicing)]
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
     let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
@@ -41,12 +42,11 @@ fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
 
 /// Remove nonstandard footer and a boolean indicating whether such
 /// footer was removed.
+#[allow(clippy::indexing_slicing)]
 fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     for (ix, &line) in lines.iter().enumerate() {
         if line == "--"
-            || line == "---"
-            || line == "----"
-            || line.starts_with("-----")
+            || line.starts_with("---")
             || line.starts_with("_____")
             || line.starts_with("=====")
             || line.starts_with("*****")
@@ -107,6 +107,7 @@ fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     }
 }
 
+#[allow(clippy::indexing_slicing)]
 fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     let mut last_quoted_line = None;
     for (l, line) in lines.iter().enumerate().rev() {
@@ -132,6 +133,7 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     }
 }
 
+#[allow(clippy::indexing_slicing)]
 fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     let mut last_quoted_line = None;
     let mut has_quoted_headline = false;
@@ -332,9 +334,17 @@ mod tests {
         let (plain, _) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
 
+        // Nonstandard footer sent by https://siju.es/
+        let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
+        let (plain, _) = simplify(input.clone(), false);
+        assert_eq!(plain, "Message text here [...]");
+        let (plain, _) = simplify(input.clone(), true);
+        assert_eq!(plain, input);
+
         let input = "--\ntreated as footer when unescaped".to_string();
         let (plain, _) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
+
         let escaped = escape_message_footer_marks(&input);
         let (plain, _) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");

src/smtp/mod.rs

@@ -2,14 +2,14 @@
 
 pub mod send;
 
-use std::time::{Duration, Instant};
+use std::time::{Duration, SystemTime};
 
 use async_smtp::smtp::client::net::*;
 use async_smtp::*;
 
 use crate::constants::*;
 use crate::context::Context;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::oauth2::*;
 use crate::provider::get_provider_info;
@@ -55,7 +55,7 @@ pub(crate) struct Smtp {
     /// Timestamp of last successful send/receive network interaction
     /// (eg connect or send succeeded). On initialization and disconnect
     /// it is set to None.
-    last_success: Option<Instant>,
+    last_success: Option<SystemTime>,
 }
 
 impl Smtp {
@@ -76,7 +76,11 @@ impl Smtp {
     /// have been successfully used the last 60 seconds
     pub async fn has_maybe_stale_connection(&self) -> bool {
         if let Some(last_success) = self.last_success {
-            Instant::now().duration_since(last_success).as_secs() > 60
+            SystemTime::now()
+                .duration_since(last_success)
+                .unwrap_or_default()
+                .as_secs()
+                > 60
         } else {
             false
         }
@@ -98,7 +102,7 @@ impl Smtp {
         }
 
         if lp.send_server.is_empty() || lp.send_port == 0 {
-            context.emit_event(Event::ErrorNetwork("SMTP bad parameters.".into()));
+            context.emit_event(EventType::ErrorNetwork("SMTP bad parameters.".into()));
             return Err(Error::BadParameters);
         }
 
@@ -179,18 +183,18 @@ impl Smtp {
                 .stock_string_repl_str2(
                     StockMessage::ServerResponse,
                     format!("SMTP {}:{}", domain, port),
-                    format!("{}, ({:?})", err.to_string(), err),
+                    err.to_string(),
                 )
                 .await;
 
-            emit_event!(context, Event::ErrorNetwork(message));
+            emit_event!(context, EventType::ErrorNetwork(message));
             return Err(Error::ConnectionFailure(err));
         }
 
         self.transport = Some(trans);
-        self.last_success = Some(Instant::now());
+        self.last_success = Some(SystemTime::now());
 
-        context.emit_event(Event::SmtpConnected(format!(
+        context.emit_event(EventType::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
             lp.send_user,
         )));

src/smtp/send.rs

@@ -4,7 +4,8 @@ use super::Smtp;
 use async_smtp::*;
 
 use crate::context::Context;
-use crate::events::Event;
+use crate::events::EventType;
+use std::time::Duration;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -30,7 +31,7 @@ impl Smtp {
         message: Vec<u8>,
         job_id: u32,
     ) -> Result<()> {
-        let message_len = message.len();
+        let message_len_bytes = message.len();
 
         let recipients_display = recipients
             .iter()
@@ -47,13 +48,18 @@ impl Smtp {
         );
 
         if let Some(ref mut transport) = self.transport {
-            transport.send(mail).await.map_err(Error::SendError)?;
+            // The timeout is 1min + 3min per MB.
+            let timeout = 60 + (180 * message_len_bytes / 1_000_000) as u64;
+            transport
+                .send_with_timeout(mail, Some(&Duration::from_secs(timeout)))
+                .await
+                .map_err(Error::SendError)?;
 
-            context.emit_event(Event::SmtpMessageSent(format!(
+            context.emit_event(EventType::SmtpMessageSent(format!(
                 "Message len={} was smtp-sent to {}",
-                message_len, recipients_display
+                message_len_bytes, recipients_display
             )));
-            self.last_success = Some(std::time::Instant::now());
+            self.last_success = Some(std::time::SystemTime::now());
 
             Ok(())
         } else {

src/sql.rs

@@ -13,6 +13,7 @@ use crate::chat::{update_device_icon, update_saved_messages_icon};
 use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::start_ephemeral_timers;
 use crate::param::*;
 use crate::peerstate::*;
 
@@ -568,16 +569,24 @@ pub async fn housekeeping(context: &Context) {
         }
     }
 
+    if let Err(err) = start_ephemeral_timers(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot start ephemeral timers: {}", err
+        );
+    }
+
     if let Err(err) = prune_tombstones(context).await {
         warn!(
             context,
-            "Houskeeping: Cannot prune message tombstones: {}", err
+            "Housekeeping: Cannot prune message tombstones: {}", err
         );
     }
 
     info!(context, "Housekeeping done.",);
 }
 
+#[allow(clippy::indexing_slicing)]
 fn is_file_in_use(files_in_use: &HashSet<String>, namespc_opt: Option<&str>, name: &str) -> bool {
     let name_to_check = if let Some(namespc) = namespc_opt {
         let name_len = name.len();
@@ -593,11 +602,9 @@ fn is_file_in_use(files_in_use: &HashSet<String>, namespc_opt: Option<&str>, nam
 }
 
 fn maybe_add_file(files_in_use: &mut HashSet<String>, file: impl AsRef<str>) {
-    if !file.as_ref().starts_with("$BLOBDIR") {
-        return;
+    if let Some(file) = file.as_ref().strip_prefix("$BLOBDIR/") {
+        files_in_use.insert(file.to_string());
     }
-
-    files_in_use.insert(file.as_ref()[9..].into());
 }
 
 async fn maybe_add_from_param(
@@ -1241,6 +1248,46 @@ async fn open(
                 .await?;
             sql.set_raw_config_int(context, "dbversion", 63).await?;
         }
+        if dbversion < 64 {
+            info!(context, "[migration] v64");
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN error TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 64).await?;
+        }
+        if dbversion < 65 {
+            info!(context, "[migration] v65");
+            sql.execute(
+                "ALTER TABLE chats ADD COLUMN ephemeral_timer INTEGER",
+                paramsv![],
+            )
+            .await?;
+            // Timer value in seconds. For incoming messages this
+            // timer starts when message is read, so we want to have
+            // the value stored here until the timer starts.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            // Timestamp indicating when the message should be
+            // deleted. It is convenient to store it here because UI
+            // needs this value to display how much time is left until
+            // the message is deleted.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 65).await?;
+        }
+        if dbversion < 66 {
+            info!(context, "[migration] v66");
+            update_icons = true;
+            sql.set_raw_config_int(context, "dbversion", 66).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1261,7 +1308,7 @@ async fn open(
                 )
                 .await?;
             for addr in &addrs {
-                if let Some(ref mut peerstate) = Peerstate::from_addr(context, addr).await {
+                if let Some(ref mut peerstate) = Peerstate::from_addr(context, addr).await? {
                     peerstate.recalc_fingerprint();
                     peerstate.save_to_db(sql, false).await?;
                 }
@@ -1303,10 +1350,12 @@ mod test {
         maybe_add_file(&mut files, "$BLOBDIR/hello");
         maybe_add_file(&mut files, "$BLOBDIR/world.txt");
         maybe_add_file(&mut files, "world2.txt");
+        maybe_add_file(&mut files, "$BLOBDIR");
 
         assert!(files.contains("hello"));
         assert!(files.contains("world.txt"));
         assert!(!files.contains("world2.txt"));
+        assert!(!files.contains("$BLOBDIR"));
     }
 
     #[test]

src/stock.rs

@@ -130,7 +130,9 @@ pub enum StockMessage {
     ))]
     AcSetupMsgBody = 43,
 
-    #[strum(props(fallback = "Cannot login as %1$s."))]
+    #[strum(props(
+        fallback = "Cannot login as \"%1$s\". Please check if the email address and the password are correct."
+    ))]
     CannotLogin = 60,
 
     #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
@@ -177,11 +179,43 @@ pub enum StockMessage {
                     however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
     WelcomeMessage = 71,
 
-    #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
+    #[strum(props(fallback = "Unknown sender for this chat. See 'info' for more details."))]
     UnknownSenderForChat = 72,
 
     #[strum(props(fallback = "Message from %1$s"))]
     SubjectForNewContact = 73,
+
+    #[strum(props(fallback = "Failed to send message to %1$s."))]
+    FailedSendingTo = 74,
+
+    #[strum(props(fallback = "Message deletion timer is disabled."))]
+    MsgEphemeralTimerDisabled = 75,
+
+    // A fallback message for unknown timer values.
+    // "s" stands for "second" SI unit here.
+    #[strum(props(fallback = "Message deletion timer is set to %1$s s."))]
+    MsgEphemeralTimerEnabled = 76,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 minute."))]
+    MsgEphemeralTimerMinute = 77,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 hour."))]
+    MsgEphemeralTimerHour = 78,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 day."))]
+    MsgEphemeralTimerDay = 79,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 week."))]
+    MsgEphemeralTimerWeek = 80,
+
+    #[strum(props(fallback = "Message deletion timer is set to 4 weeks."))]
+    MsgEphemeralTimerFourWeeks = 81,
+
+    #[strum(props(fallback = "Video chat invitation"))]
+    VideochatInvitation = 82,
+
+    #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
+    VideochatInviteMsgBody = 83,
 }
 
 /*
@@ -331,10 +365,10 @@ impl Context {
         let action1 = action.trim_end_matches('.');
         match from_id {
             0 => action,
-            1 => {
+            DC_CONTACT_ID_SELF => {
                 self.stock_string_repl_str(StockMessage::MsgActionByMe, action1)
                     .await
-            } // DC_CONTACT_ID_SELF
+            }
             _ => {
                 let displayname = Contact::get_by_id(self, from_id)
                     .await
@@ -406,7 +440,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_stock_translation() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::NoMessages, "xyz".to_string())
             .await
@@ -416,7 +450,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_set_stock_translation_wrong_replacements() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert!(t
             .ctx
             .set_stock_translation(StockMessage::NoMessages, "xyz %1$s ".to_string())
@@ -431,7 +465,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_str() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx.stock_str(StockMessage::NoMessages).await,
             "No messages."
@@ -440,7 +474,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_str() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         // uses %1$s substitution
         assert_eq!(
             t.ctx
@@ -453,7 +487,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_int() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_string_repl_int(StockMessage::MsgAddMember, 42)
@@ -464,7 +498,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_string_repl_str2() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar")
@@ -475,7 +509,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_simple() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
@@ -486,7 +520,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_me() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(
@@ -502,7 +536,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_me_with_displayname() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         Contact::create(&t.ctx, "Alice", "alice@example.com")
             .await
             .expect("failed to create contact");
@@ -521,7 +555,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_add_member_by_other_with_displayname() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let contact_id = {
             Contact::create(&t.ctx, "Alice", "alice@example.com")
                 .await
@@ -545,7 +579,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_grp_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_eq!(
             t.ctx
                 .stock_system_msg(
@@ -561,7 +595,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_stock_system_msg_grp_name_other() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let id = Contact::create(&t.ctx, "Alice", "alice@example.com")
             .await
             .expect("failed to create contact");
@@ -576,7 +610,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_update_device_chats() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.ok();
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 2);

src/test_utils.rs

@@ -18,43 +18,64 @@ pub(crate) struct TestContext {
     pub dir: TempDir,
 }
 
-/// Create a new, opened [TestContext] using given callback.
-///
-/// The [Context] will be opened with the SQLite database named
-/// "db.sqlite" in the [TestContext.dir] directory.
-///
-/// [Context]: crate::context::Context
-pub(crate) async fn test_context() -> TestContext {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let ctx = Context::new("FakeOs".into(), dbfile.into()).await.unwrap();
-    TestContext { ctx, dir }
-}
+impl TestContext {
+    /// Create a new [TestContext].
+    ///
+    /// The [Context] will be created and have an SQLite database named "db.sqlite" in the
+    /// [TestContext.dir] directory.  This directory is cleaned up when the [TestContext] is
+    /// dropped.
+    ///
+    /// [Context]: crate::context::Context
+    pub async fn new() -> Self {
+        use rand::Rng;
 
-/// Return a dummy [TestContext].
-///
-/// The context will be opened and use the SQLite database as
-/// specified in [test_context] but there is no callback hooked up,
-/// i.e. [Context::call_cb] will always return `0`.
-pub(crate) async fn dummy_context() -> TestContext {
-    test_context().await
-}
+        let dir = tempdir().unwrap();
+        let dbfile = dir.path().join("db.sqlite");
+        let id = rand::thread_rng().gen();
+        let ctx = Context::new("FakeOS".into(), dbfile.into(), id)
+            .await
+            .unwrap();
+        Self { ctx, dir }
+    }
 
-pub(crate) async fn configured_offline_context() -> TestContext {
-    let t = dummy_context().await;
-    t.ctx
-        .set_config(Config::Addr, Some("alice@example.org"))
-        .await
-        .unwrap();
-    t.ctx
-        .set_config(Config::ConfiguredAddr, Some("alice@example.org"))
-        .await
-        .unwrap();
-    t.ctx
-        .set_config(Config::Configured, Some("1"))
-        .await
-        .unwrap();
-    t
+    /// Create a new configured [TestContext].
+    ///
+    /// This is a shortcut which automatically calls [TestContext::configure_alice] after
+    /// creating the context.
+    pub async fn new_alice() -> Self {
+        let t = Self::new().await;
+        t.configure_alice().await;
+        t
+    }
+
+    /// Configure with alice@example.com.
+    ///
+    /// The context will be fake-configured as the alice user, with a pre-generated secret
+    /// key.  The email address of the user is returned as a string.
+    pub async fn configure_alice(&self) -> String {
+        let keypair = alice_keypair();
+        self.configure_addr(&keypair.addr.to_string()).await;
+        key::store_self_keypair(&self.ctx, &keypair, key::KeyPairUse::Default)
+            .await
+            .expect("Failed to save Alice's key");
+        keypair.addr.to_string()
+    }
+
+    /// Configure as a given email address.
+    ///
+    /// The context will be configured but the key will not be pre-generated so if a key is
+    /// used the fingerprint will be different every time.
+    pub async fn configure_addr(&self, addr: &str) {
+        self.ctx.set_config(Config::Addr, Some(addr)).await.unwrap();
+        self.ctx
+            .set_config(Config::ConfiguredAddr, Some(addr))
+            .await
+            .unwrap();
+        self.ctx
+            .set_config(Config::Configured, Some("1"))
+            .await
+            .unwrap();
+    }
 }
 
 /// Load a pre-generated keypair for alice@example.com from disk.
@@ -77,20 +98,6 @@ pub(crate) fn alice_keypair() -> key::KeyPair {
     }
 }
 
-/// Creates Alice with a pre-generated keypair.
-///
-/// Returns the address of the keypair created (alice@example.com).
-pub(crate) async fn configure_alice_keypair(ctx: &Context) -> String {
-    let keypair = alice_keypair();
-    ctx.set_config(Config::ConfiguredAddr, Some(&keypair.addr.to_string()))
-        .await
-        .unwrap();
-    key::store_self_keypair(&ctx, &keypair, key::KeyPairUse::Default)
-        .await
-        .expect("Failed to save Alice's key");
-    keypair.addr.to_string()
-}
-
 /// Load a pre-generated keypair for bob@example.net from disk.
 ///
 /// Like [alice_keypair] but a different key and identity.

test-data/message/gmail_ndn.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a1c:b4d7:0:0:0:0:0 with SMTP id d206csp3026053wmf;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Received: by 2002:a5d:4651:: with SMTP id j17mr19532177wrs.50.1589819005555;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1589819005; cv=none;
+        d=google.com; s=arc-20160816;
+        b=IZbNnzzuYzTFuqfuZwpd3ehqpYYGpn31c8DsfGbQ8rpbS0OTTROkVYvihQl8Ne/8X/
+         brEWsrcmaCh9WpFMzpI+cp/TY39uusnI6qdp5rcgrFmFgoANtwf3TBBj1+f7wBPn46BP
+         dQOUsg/J8KVfvzVgvL1x4uyJ0m9QirDgJeJ/BvrswbTleRQK7oY3fIireUCDxj6r2lCB
+         1Z0TKw1mgIb1LiFMZz8kvCNn3R4KSFnwS8rIju0hYwnsioNiExVQgumXL+RVkEZ9BMzf
+         UdoWIAw3VW+MOZFTpfLCEfgIPtLg/gtE0Q1P+a3KKpi8dkPiV2n6DGMecy9lTLtdhCXt
+         pnaA==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=nNP0DktrSjdBaFfhhoDi2O9KVKM0iXE5ZgubQ0q0ff68Z6Ke7c8dDBXEsZoToI0s4Y
+         w90KyJFpgMJLFmP3iVDRqCfohi2y1HGdWg5VXQPTvzM7+YozZRlbNNV9UsuyRY91CXrJ
+         a2XREBgB+LPMGQivwcHtUMZfyNv/4uiwWivk+92ySNDhxqOiDt4R5Jak/7RkZMFwQpsE
+         JGwk6asM6VqZlihkF24lKv3pPaob6feyX3wD5N0+Mqiy1kQTj2JkpQk6nkTmdf0gapZe
+         fOhU1NkbNfbuS3U7m2gEUiyktE+MhV/MgAzgBhm9bgNt2gQLVWju8rHkPndfv1PDmEkC
+         FsYQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id s18sor5584435wrb.25.2020.05.18.09.23.25
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=dPisws+O;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=dPisws+OwGFyOy0a612XYZgvz5T71GcJRJtU068/Tce8vN/+ggIQtUsZnZtsphe71v
+         2NvfP9ULxR4cXvomTvhrYAk19KdxN/S7SeyBbmXv3x/tg+DBVCmmPS/6RXrcl6Ms3Hkw
+         uPFQ9S3KcvHe/2bcb5LSTA/stIP4tuxxAXvsX2j+MjPYPWKAl50jkSbWK98U0Q0U+MTl
+         pKaaC9s9iEBafac8BFZCy4DfpumKlemNEyRa3cSV2hw+DYHKA5peModrK1A2tcsfstFF
+         rZi8yF/D90RIFbE04DI2QCxB3trsChNF1aYF06aSzI//wsfM1+lb+uGPi0YVkw3n4HrX
+         Xw4w==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=5xjZvcHbEGbMY0K2QB+3U6tpm1L1LAVv5h1pd4YXDEE=;
+        b=A/NCOtgbpA7VzB1G7ZFo8TA2FfrjuqjGdwMrJr3yXe21FrBFwzssprJwOkynqoVLkK
+         iJU7uMF/KTcQPDEmOLFThzFfe5GCx7eJtZPhwY+FbBlC5sq4I55/xaQLd0gOZ1BYXwMn
+         2bk169d2aoukbaLbGSQZF3d9atd+/e48YzkRxpmUoLcrWk2LcHAeQIG7SgT9pfX5DKPr
+         VpxM5/GMVEBbTRhBIWCeVSfpYCs80l0xEeTC3/B5lzpzMVDE8QCW6Dwh75b4Tb2K6yru
+         Zsy5ZpRmwv0wrkrb2vM+pl4IMkaF7s8XosIvlIT++fQV5xDFItT4atpykZvSDB92RKV0
+         8lEA==
+X-Gm-Message-State: AOAM532RG/PT3ChZHBCDORGLtAjKvX8TGBuOy+AxrnEaJT6v1ieb+VV1
+	+ejly+/6UthxHYlkOJYAszCSgL4dKVFotoVaN7LhEA==
+X-Google-Smtp-Source: ABdhPJz6veVKWhomCL4gK+whrybuMzHCDCq8AowgQvi7sobpMoM/k9CDw79jo1j3OUcTz6MEeUYLxEXuNIuu4zyoS7kVtsUYryGFHAI=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545185wrv.159.1589819005394;
+        Mon, 18 May 2020 09:23:25 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000012d63005a5ee9520"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a5d:5183:: with SMTP id k3mr13704211wrv.159; Mon, 18 May
+ 2020 09:23:25 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ec2b67d.1c69fb81.213af.67a5.GMR@mx.google.com>
+Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+In-Reply-To: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000012d63005a5ee9520
+Content-Type: multipart/related; boundary="00000000000012dc0005a5ee952f"
+
+--00000000000012dc0005a5ee952f
+Content-Type: multipart/alternative; boundary="00000000000012dc0705a5ee9530"
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+
+--00000000000012dc0705a5ee9530
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--00000000000012dc0705a5ee9530--
+--00000000000012dc0005a5ee952f
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--00000000000012dc0005a5ee952f--
+--00000000000012d63005a5ee9520
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+X-Original-Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp
+Last-Attempt-Date: Mon, 18 May 2020 09:23:25 -0700 (PDT)
+
+--00000000000012d63005a5ee9520
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=a185ogBcMzF9whNVWvuyUoUunNZk3Vc1kEIFmPkX0IxLpAFcI+fOQajOSromGl7Oyi
+         yecLwQevpww2Xd0XjZ3UkZvrI9m9koRmh0QeoHvgTRORiVwj08+PVc3N4F9bCO4w9i0J
+         ir7SSsJqBCDovoIFSFDyNa64vs6Nxno0cH/DaPG7pVTdD+3jfB7nLXIsMQYeX+1eP6rB
+         UhKxH82r7Mh9CI2PWDQpVtGj63AMUEyHgE9Ou08PWbbKjrQOasoG3Tw8tB1GoN1XYssM
+         rxOTgWEoTiduZ35AUH6h+eChOn9OHuI3SPECcVob70Qndayia3dMKfHMO6sEx9J0Wpic
+         29vg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=gtlm3j0shCgZYOVxUt74zkQ69Zq+GTQeHeXLfMlrhlk=;
+        b=miGIfL5BgnkD3wQvS34RtGwRRoh+8gJT5sFFfdX/hVyG/dvjXfdwP4yyNWr8ox8iY2
+         BLlahS4y4VGcbG1e2aYjurnWNytGu6utQcZax/uUngJ0bTOwXW1VaIiEZtqd6gTV+8d/
+         rrfQ459+4vXqIoQf0+Oi/U6dWwgJvPPjjRiToWdF3vIJE8R1iTRdZbW4lkgxSADbmskg
+         noT/gWGWblHtR6uuGuKGJ3bkhJKCBnjavKh0LlbWEeFBZfmVNPRvzEFWHjBDdu5wvSL5
+         0QJ+Qn0Orfn5CJuN3xPfzT1S2rI2iYZx37KX9zyMnZEx0ilkTYqCtBPWkrXRYDSXcxYS
+         Y1ag==
+X-Gm-Message-State: AOAM531vhwpXiK8M12286dOJx0Q5fBl9ZaH6BJKts93GoxvPv0xdryP0
+	jg9wYmoP5MUHudsxAMCYDFsCUMVx2PEywyIsaQqklw==
+X-Google-Smtp-Source: ABdhPJxlVJtTODM3pZZSTbbpAAAQRU8XbmuosDF9fgQZmVwxGZSzRWl22o+moppVRU/r8xMAyf0r3+qXwEBe1vZfjZo=
+X-Received: by 2002:a5d:5183:: with SMTP id k3mr20545162wrv.159.1589819005034;
+ Mon, 18 May 2020 09:23:25 -0700 (PDT)
+MIME-Version: 1.0
+From: <alice@gmail.com>
+Date: Mon, 18 May 2020 18:23:39 +0200
+Message-ID: <CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com>
+Subject: Kommt sowieso nicht an
+To: assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000000d652a05a5ee95df"
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/plain; charset="UTF-8"
+
+Wollte nur was testen
+
+--0000000000000d652a05a5ee95df
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">Wollte nur was testen<br></div>
+
+--0000000000000d652a05a5ee95df--
+
+--00000000000012d63005a5ee9520--

test-data/message/gmail_ndn_group.eml

@@ -0,0 +1,242 @@
+Delivered-To: alice@gmail.com
+Received: by 2002:a02:6629:0:0:0:0:0 with SMTP id k41csp368502jac;
+        Wed, 10 Jun 2020 05:17:57 -0700 (PDT)
+X-Received: by 2002:a6b:1448:: with SMTP id 69mr2898530iou.83.1591791475733;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1591791475; cv=none;
+        d=google.com; s=arc-20160816;
+        b=a0vSKJPbMtGYFnuk1ye/gnnV00Zvva4OOJTMOyfm13xMJD0YAhzGVfa7Z+wn5sQ8dw
+         VAxpmDHCkjp4jol0C1iutiq2Nl0qma819oFPuuoMLLatKQXHpo8Jt+sL3MnwNR7J5bZC
+         1c6Fjk75EIsRWhJd1HCkm44A6UYHxqqsTnzQCaNiHbjsRsvbggxwlMGSrZ4silxqSDvo
+         Pzd/YDLCvsnZNSNIjIckKAwtGmY6sXctZ+JnOTykXAyL32Milfwy1vRL9xm10Q14biTR
+         +qaIQp4E6WE63g1WHvfAjs0Dru7DalUh4GGl/NAwqVhY1gVyRD5E9/nODyHAfxjvaxDD
+         4sMw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=OJbgbrktMKyczw25z/ib7lSdRX80PEK3Myh9fj4q6mDlXmPPv//Gv069znRQ4QbadM
+         HUXZH0WLMZcGyqI6SvGL/noxQ1O8yP0FYJJKTkoX0Gk2hHzfaE3x1scOP/o2FMMQXIFm
+         S4CgGBD6HHzBJYj/rSL3gzqLzx1Id/z5kTeDvH2cn8JJAcCE2q/nhjTyWUb87geoNlDJ
+         A1HRrLHK/0JOyRjHfg2zZCqIvSi1xmpiHStMyL9mfVyrQs98tsPxaOkJHjLplFARoPlr
+         mmmDvsFg7MPvFqkkANzz4JDHidnfKRULCgnrVj1yTU66UagUpQEGjZqz8/99YuU6nt1t
+         81sQ==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id w14sor16686480iow.23.2020.06.10.05.17.55
+        for <alice@gmail.com>
+        (Google Transport Security);
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b=aO4aNy7C;
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=aO4aNy7CUOk9O4Jnsue/DvMFY6Ph0C34AbpoxJH+mLZpOmt/KYGCGYWgunZgamF15U
+         Vm8JY5yLKGwkTz2m3abDnKNP4fpl6zeZ5fyk5LvXH2Jema0iocHai6pJZBoFGPnonNmd
+         MscTf1sEltbOxwfOmM1BRHX34c1jW0+8Yd2+Nhg2DPvzuq1brOVin6bUV4VX5EeeuNqT
+         ZTewjJVPmO/B5NQhdpG81FO5w4hKSQ/VzZXnap2thMf3gOmnaoR+tbsnOIAiklcLdJ7b
+         57SKUwI041pwSmh9dffs0STl2GvMRSJyGCtBqMnzXgflqoGTcnPflWgR3LXHM/MIA0q8
+         WqRQ==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=XaR1H4XeD+InO7mULPJn53omDGmxN+KG6DbSxyyErPM=;
+        b=iORAzNvXegQ8oSp4RYb/S168muAiBox769seMk49kDBIvXwI+N8P4mUZq/zDi8DmQd
+         +wlLzVzowQq6EofiSpjOJWT9IC/k8otk15PMGtgHE4BGSSeKn7L30d3ocQS93HzYnLmA
+         VBlHBdFTKrsfKhe2+CQyCosTDGRpbkQLuRRyhxChEq8ltvaOHgbu1+eCeb9PsPuh6OxH
+         kvTHJZeA9A+eLOl26pBmqGIWkr7FlYW0wI6YPoEs9WXX5LSFOQs6fm/9l366eIR7IFFI
+         ihX5LrZl/Cf0lwwYX7fqIMgnHy1K+QnKuEb+dRQGqLbxdIEls9bXIF98iPQVkEWzgSZy
+         ip8Q==
+X-Gm-Message-State: AOAM531ahfHE6oS9/nuni8pNf9bwC+DXAcaLV0owBwNCj9kcTPLCCNhX
+	W1JNciK0ivEIVB4dgiyLE/5K7iKbEznQhqyG9Bi1QA==
+X-Google-Smtp-Source: ABdhPJygljUXswH0ycJyHmXVthi5IjlDvP8QdYlMdHUPKEtgIZeUk69Acti5LnswGhg63T9/L0PuGZGBM5XE5BsP0mMNNDRZyt+DgnE=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990163jab.84.1591791475516;
+        Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+Content-Type: multipart/report; boundary="00000000000074432a05a7b9d512"; report-type=delivery-status
+To: alice@gmail.com
+Received: by 2002:a05:6638:101c:: with SMTP id r28mr3059870jab.84; Wed, 10 Jun
+ 2020 05:17:55 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <5ee0cf73.1c69fb81.6888.c2f4.GMR@mx.google.com>
+Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+In-Reply-To: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+X-Failed-Recipients: assidhfaaspocwaeofi@gmail.com
+
+--00000000000074432a05a7b9d512
+Content-Type: multipart/related; boundary="000000000000745e0805a7b9d51b"
+
+--000000000000745e0805a7b9d51b
+Content-Type: multipart/alternative; boundary="000000000000745e1705a7b9d51c"
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Die Adresse wurde nicht gefunden **
+
+Ihre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+
+Hier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser
+
+Antwort:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+
+--000000000000745e1705a7b9d51c
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Fehlersymbol" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Die Adresse wurde nicht gefunden
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Ihre Nachricht wurde nicht an <a style='color:#212121;text-decoration:none'><b>assidhfaaspocwaeofi@gmail.com</b></a> zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">WEITERE INFORMATIONEN</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+Antwort:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--000000000000745e1705a7b9d51c--
+--000000000000745e0805a7b9d51b
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--000000000000745e0805a7b9d51b--
+--00000000000074432a05a7b9d512
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Arrival-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+X-Original-Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+
+Final-Recipient: rfc822; assidhfaaspocwaeofi@gmail.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser h20sor9401601jar.6 - gsmtp
+Last-Attempt-Date: Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+
+--00000000000074432a05a7b9d512
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=gmail.com; s=20161025;
+        h=mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=eyr60XbgOrgHoZFpRYzw9WQIR7aEBaYKWhiEcqdnugB+hn0W2KVcTkKiL2C6zSF+jh
+         l+lM+dNZZTUcMqWx4kVgTVtqwUNea8OUqe+WLqx04ULwdKZn1okbKYovaiavCLKOKDnf
+         ZP5mNz3Ka/ywpCGoq8rdgnXc7NunnkWeaBpYY/BWOmLU4WNXX8zS7etXXhQE4YPQEJT4
+         Sh2o/YIIjDLncJFMyE+25n3tbd2mIoLt4sjaCHE5ibm9w7zojyHM+LDCQ37cM74FEAAa
+         88KTn0gSnCFBCfojhfxOH78CpySHG3FFfTlpCefwP2A5J9MQlb6QdSVa9STYSx3IntJ4
+         L7Tg==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:mime-version:from:date:message-id:subject:to;
+        bh=Y1ylbv3YC5frF/LtF2it4tQQ0OVZstDdWqivvggIOB0=;
+        b=pBL4/bKUDw5E2zo1uR2Tl69h2iTlMgIAcnzQgodPCbU4jZ9kH+F5H9rfbzXCjT06J7
+         L72SYpdfgc5fOwM4GhRcdYnyK3wiXQ8ugpL19nbYt2iWo/vRF3GidawXXDGb2GUYpkzX
+         1Mz531cy2/HOsmQbUQ7304KV+OUghtcg8eLNnFuhQch7n12Kk3yy3AOzjrLoktcdgIsy
+         /HxBjyut0Au+A2t6si+PVwTHvC647a0BioeV0tUYLigzu3/jgP9Hb8eRZaXTX5VC6iZi
+         9QMH/+rXp05IK7OpGWh22xDpeV8CDkQ2sLFaBhKxtJ+nYoerM64t8EJXBBsVQb18ojGz
+         pW/A==
+X-Gm-Message-State: AOAM5330q6kn/TKataMNEVigNfNdr/xii/PQgHXzJyMbwLvsETlNfLoy
+	1rM9JBIGrcHeEDRx4qhZfl5S4bircceU7c3i6Fyn2fRO
+X-Google-Smtp-Source: ABdhPJwysG+S90b/g+9mK7LgeHhmJTBowst6JMhL16+a0coTi7P1NVp9jjaNHJfhvhLodYG6eHIvWdbQGJnAP2brEzI=
+X-Received: by 2002:a05:6638:101c:: with SMTP id r28mr2990137jab.84.1591791475066;
+ Wed, 10 Jun 2020 05:17:55 -0700 (PDT)
+MIME-Version: 1.0
+From: Deltachat Test <alice@gmail.com>
+Date: Wed, 10 Jun 2020 14:18:26 +0200
+Message-ID: <CADWx9Cs32Wa7Gy-gM0bvbq54P_FEHe7UcsAV=yW7sVVW=fiMYQ@mail.gmail.com>
+Subject: test
+To: bob@example.org, assidhfaaspocwaeofi@gmail.com
+Content-Type: multipart/alternative; boundary="0000000000006d8d7d05a7b9d5b3"
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/plain; charset="UTF-8"
+
+test
+
+--0000000000006d8d7d05a7b9d5b3
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">test<br></div>
+
+--0000000000006d8d7d05a7b9d5b3--
+
+--00000000000074432a05a7b9d512--

test-data/message/gmx_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Received: from mout-bounce.gmx.net ([212.227.15.44]) by mx-ha.gmx.net
+ (mxgmx101 [212.227.17.5]) with ESMTPS (Nemesis) id 1Mr97m-1jC6Y01o86-00oEqk
+ for <alice@gmx.de>; Tue, 09 Jun 2020 14:35:30 +0200
+Received: from localhost by mout-bounce.gmx.net id 0LhiZF-1jDTj11ZoH-00msO3
+    Tue, 09 Jun 2020 14:35:30 +0200
+Date: Tue, 09 Jun 2020 14:35:30 +0200
+From: "GMX Mailer Daemon" <mailer-daemon@gmx.de>
+To: alice@gmx.de
+Subject: Mail delivery failed: returning message to sender
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+X-UI-Out-Filterresults: unknown:0;V03:K0:O8yx6kuPaGQ=:0wIDPNXEr0wX2oNsLnXaWA
+ ==
+Envelope-To: <alice@gmx.de>
+X-GMX-Antispam: 0 (Mail was not recognized as spam); Detail=V3;
+X-Spam-Flag: NO
+X-UI-Filterresults: notjunk:1;V03:K0:QcE43EBhMmU=:IC5vvzi9jhPS/698Wuubzw1Q4N
+ h87X9j9B3CBN0ZKXB67KepwyNHmh9pxmFIUOMimylv7UK9np+j3X55roOd0nX9BmaaZ3Twvqf
+ UaSsxmyU+cNr6m3+oOb3udJBLe2pJEZDk1cOwACb5NXzYPSaIj4APfGCyvrzIx3FGkNuScNBb
+ tCbbKUJ0GB/VmJLB34XfF6dNN+Iwv9IQ9Yrvw/VXv9vWKsi3qRGGUt3yRw5jUKhQlBY21Pnoq
+ m0LqoMbAKfH1tKEQ/5TymH1ei50YKyWzZ89ISkQwkbYLaqN+6meGACpY18j43VCU9Fk4WQR7y
+ 3XvBYh2CO0CnCn+M9VsnasYag2sNrySe9nzyKfRTaxEg8qlJtl7kS4GX/FsxhHPavkqnU62Gl
+ 9V5TxIG7tmIR0Bf11sPzG/WGegoOHxrfz+qYR81llLMOHznpdDRKjsYDtO/rFBGZzYTiCZsrW
+ dZPVXV25SVcrDGZOaop3JoCbglmXLcSLLhmfE5MzyJEGte3I+6EiZJNeIe8qN3wMDTsRtJL9S
+ J6b2F/5/kTGVAWnXtNlf69BholCrxvjC4Snt3Xjc+7WIO8iw2c5YjmWy+4bAwd4uWll529hZd
+ 6pUYGwjFRnKleivCaJIt7DqbvbE7GZSbQH8fXm3zYqYTrrxiWdlykdoOA1OGbeM06RHJt3mJB
+ osZPU8BZKt0OiBOW64vg6gyAsNC0f02EA7dvRWYgFYqlSogfWZQIOKDKibMVHpIaA0foXg4BG
+ TEQDlsTIL0n2WC9WVqkMdm6xUXHgpArCrAsUhw3mEqPywEfJeBHn60tP2vQ9+pDIQAj5dQCDV
+ y96qSiCX4p31HfrWwAXB9mHfl4OO/tPcKUGBclj2rZ/NMc4O+7yDedLWXQnRzQExfOJLBbBh3
+ xgiNlWFHvDLn0pKG9EI1+3wJ7m2GF2jzDtbQTBv9z26DuAq5WbHZHupzeyfP7VCVXcKuB6sG1
+ 3+LWcdYtcXfqT58HwcvDLwowC4uJpiHfHwtVdiGMtHnmYLysp0V425g+vofQfNzBgR3d9JC15
+ G+HS44o6x6Legm6KnHYH3k0KhR7fgcgswJv/S+I/ryppUhGb2jezVZIUzgvAplzIUDAWnrHdF
+ KVqZ5wBJ0acShIfgMlsIxnBmcnIQ4R4jq3zAyj4XTFxVUFanU8ySiXubxV5PzJqj+GsVa2sjx
+ 9n/xQRJLwgMC4BYqzP6lEPwg/g5AneDAnl7ZlcQPC4SCMblC8N1KZyyIDTXPOI/o4lfdMYb9P
+ 7DmBp2S8aA2yuDe5XT20OmX3kVWeBOsBaAGvVFpIn7gwIDqnFh9WSMP6mkCwfChN3D1yLquYB
+ KAODgRZV5lVNmK+eOjW8m2oiRxmfxrjXLtw5PEhn3RkiRN4HnoePJeoYC7SG4EUwg+wYPu3M6
+ exP/YigoE6bjuBS5d0imUTRDMiwg469GyrFo1J1GkRVvj3lXSF4Nt11j6waqu1l0ReDYfU+QM
+ EMPGLEh1vRChCaqz4L7YI5FlSAVXxfmst0JRyE4k5r9CToEbZuYlPQ+jbcvptwBSaqzMb/YfT
+ cCrU2RWHUfmYIy8x8A/t8ScRbYPzs1lTK3yn1hYeXpw8Fgkip6DIIXAJwUUp+2SLcELICIo+p
+ uumMN0P/OZHH3V/hZ0dPr9xsYi7gdd/vyRIRPUwiL1rSp2WJGi+w4atun8kQBgnbAznRObDh1
+ 4zzKhApX9jo5gtFN6640QDEI5KpsMuPoty4rj9OK163ntKWGR451n+5ZX2FilTlpZYlIPO9Hy
+ SrjHzBog4texceR1OLh4pb/WFB0XFSjQchPAltXCYQFs82aDDk/A0nOPk=
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of
+its recipients. This is a permanent error. The following address(es)
+failed:
+
+snaerituhaeirns@gmail.com:
+SMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please
+ try
+550-5.1.1 double-checking the recipient's email address for typos or
+550-5.1.1 unnecessary spaces. Learn more at
+550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21
+9 - gsmtp
+
+
+
+--- The header of the original message is following. ---
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=gmx.net;
+	s=badeba3b8450; t=1591706130;
+	bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=X-UI-Sender-Class:To:From:Subject:Date;
+	b=NwY6W33mI1bAq6lpr6kbY+LD2hO9cDJBItTgY3NRIT94A6rKTVlSmhFM3AxYgFnj0
+	 Db0hncsNRDqcdtRoOo8Emcah5NJURvEQohG37lkug3GqneB4+FNTdYCeQbOKlZn6on
+	 pYYD/T9CmeL2HG3+8voeBjZIUenyXrF2WXG37hFY=
+X-UI-Sender-Class: 01bb95c1-4bf8-414a-932a-4f6e2808ef9c
+Received: from [192.168.178.30] ([84.57.126.154]) by mail.gmx.com (mrgmx005
+ [212.227.17.190]) with ESMTPSA (Nemesis) id 1MKbkM-1jNoq60HKm-00KyL2 for
+ <snaerituhaeirns@gmail.com>; Tue, 09 Jun 2020 14:35:30 +0200
+To: snaerituhaeirns@gmail.com
+From: Alice <alice@gmx.de>
+Subject: test
+Message-ID: <9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de>
+Date: Tue, 9 Jun 2020 14:36:10 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+X-Provags-ID: V03:K1:7awoptmynaF5MqxCAincdsa7zFCwQFNkFb6xRxigK06uEiGN00b
+ Mv2wJU91CEd4mvCCWzrTtaWLZDLH8pjAWaT4+HvYIUbpwNx2jC6WHTppkYYRMVJnm4lG9pr
+ SUx1OlIcp0kbsnl3mB9xYNFwm9jzpR9Kx8QEHwIbZiiSFBcH56498UGQi//kKXVMos8C14o
+ I7cmwYmr8xB09DwLMKXfg==
+X-Spam-Flag: NO
+X-UI-Out-Filterresults: notjunk:1;V03:K0:NWQAUbhAkBc=:yAaolOnVCDWEZhgUwwvtEs
+ wXbSJ/GfMvDRpCkYpFBvHXOTpGm6hjdjQ0vLK2hvu/Hz22UdlWbIdc1J2oO9S5U20mIdc+1bS
+ TPSSpqPFc7ICPx4Wbvv2SEp9ZqH2q7ORC52UvUWfI6OjAJEPDNrXQFdUiZAa72hLj1NPeG6Qi
+ 4AbL2HwLfJ8s6TeOCm6TXRRuD+w1o/ASFOqQmoao2dFyZ2BaoAgOKPKxXYfwVGceuUygpchyS
+ 0d2bZYOXSLR+6rUYevjZAq1OCi9AIC6/wlkOe5yIRk4gJFMfPauaICsdnq3uZ9ikCAX83VWun
+ PJVMxTLTP54lgo2h0jMBX3uKk10+/wzXWplllxX9NnSa3x1V28n6raslNF0IoC6Pm72kC6Jzr
+ GkC22viCm3/Y4uHlPMOXbY5WFrQe/D9GKeJeXBLoGciNwIFkUG12a+iqWtoT+h5HVObTW8LxM
+ +UtEl97nAwxYSM+sGfIpasRpZc7r/SgN3JWGO9R9WaXpW4Cc1dH7RI+hzuZDsDUBEGTUTVPDo
+ 0SvjKHiJ6sUqGDyfv4HUgVutus6EYP27LALND4ekfom2DPRFopZhbtV5fZT7CL1Q1NogU7tYf
+ /FdmR1T1J1zCAZSFvyR5LBkfglZlHzgdnTF2heuxyqKq4dm0hnLFSULB1+CVWsg8hzrruvO5q
+ XzA6qIhBQUZmWo7wBpqpkBPxzjgTGGtXc5y5e6+crxYbbuQdnUWEnyw2xI4d6pJPqtHDA2/vT
+ ZgvNDUGceavTR5Rtyb14hhX4Q6dWK3ATy16j4hs9Aq+q/IKyVAX3A5nFYbJRIz+2YnoLr2YOa
+ IrScEorXjvTxjw+aBy73SZBe2REPzJ+O6k7chVrYjV9Q28FiGVuRYJYxWw/59Pes7IAbmfQBV
+ 4vqGCQQr4eG78gVwjw+SNdp4/6jdNkIHDqR4XW9id/r5wYxKKj4UUkSor3/+h9Rd9srh+GApy
+ uOxw/ejFvbRcxFIjvadpq1KLnO7nM27nJ4lp44ul3i7VUGefLM/45TCsuds2HM1iQWhPFQ54y
+ SA5sYjf73EUJdkHchaf5i+4uSOmbOWQ4Yvmd8+IoyoXAxvEzY2Xh53nWi8ZPY1Tu4Bw8GRrz7
+ L+VK0QiWCg3/hM7wRlFFyshmMrScMk5fOf9ynqd0JbHB7u+n4/GUwx3im/w8+NgSd3YOz7wNU
+ KD1snDWoMUO8f23Ik1Osym688OLWNwKYT+mZbMIMXcz1fB+olRZn4czMhN5DiSb8hyOxRI8NE
+ PNfaoN87CXiRkgazV6U1eiRkfcK2AvI7zOJF1tclUHZ9awyYoXtxfEzZ+J/2TCXiC7V2iSkUF
+ EjwgPxlJmccccjsxc46v1ajnTxLo0tJbZ0+DJXWkCgQ0d/iiScQ=
+
+

test-data/message/posteo_ndn.eml

@@ -0,0 +1,113 @@
+Return-Path: <>
+Delivered-To: alice@posteo.org
+Received: from proxy02.posteo.name ([127.0.0.1])
+	by dovecot03.posteo.local (Dovecot) with LMTP id zvCFJRzX317LGQIA+3EWog
+	for <alice@posteo.org>; Tue, 09 Jun 2020 20:44:24 +0200
+Received: from proxy02.posteo.de ([127.0.0.1])
+	by proxy02.posteo.name (Dovecot) with LMTP id mhNkNAnR316xBQMAGFAyLg
+	; Tue, 09 Jun 2020 20:44:23 +0200
+Received: from mailin06.posteo.de (unknown [10.0.1.6])
+	by proxy02.posteo.de (Postfix) with ESMTPS id 49hJtv3RRcz11m7
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+Received: from mx04.posteo.de (mailin06.posteo.de [127.0.0.1])
+	by mailin06.posteo.de (Postfix) with ESMTPS id 6935920DD2
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:23 +0200 (CEST)
+X-Virus-Scanned: amavisd-new at posteo.de
+X-Spam-Flag: NO
+X-Spam-Score: -1
+X-Spam-Level: 
+X-Spam-Status: No, score=-1 tagged_above=-1000 required=8
+	tests=[ALL_TRUSTED=-1] autolearn=disabled
+Received: from mout01.posteo.de (mout01.posteo.de [185.67.36.65])
+	by mx04.posteo.de (Postfix) with ESMTPS id 49hJtv001Vz10kT
+	for <alice@posteo.org>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Authentication-Results: mx04.posteo.de; dmarc=none (p=none dis=none) header.from=mout01.posteo.de
+Received: by mout01.posteo.de (Postfix)
+	id DCB6B1200DD; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+From: MAILER-DAEMON@mout01.posteo.de (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@posteo.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="B39111200B9.1591728262/mout01.posteo.de"
+Content-Transfer-Encoding: 7bit
+Message-Id: <20200609184422.DCB6B1200DD@mout01.posteo.de>
+
+This is a MIME-encapsulated message.
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Notification
+Content-Type: text/plain; charset=us-ascii
+
+This is the mail system at host mout01.posteo.de.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550
+    Requested action not taken: mailbox unavailable (in reply to RCPT TO
+    command)
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mout01.posteo.de
+X-Postfix-Queue-ID: B39111200B9
+X-Postfix-Sender: rfc822; alice@posteo.org
+Arrival-Date: Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+
+Final-Recipient: rfc822; hanerthaertidiuea@gmx.de
+Original-Recipient: rfc822;hanerthaertidiuea@gmx.de
+Action: failed
+Status: 5.0.0
+Remote-MTA: dns; mx01.emig.gmx.net
+Diagnostic-Code: smtp; 550 Requested action not taken: mailbox unavailable
+
+--B39111200B9.1591728262/mout01.posteo.de
+Content-Description: Undelivered Message Headers
+Content-Type: text/rfc822-headers
+
+Return-Path: <alice@posteo.org>
+Received: from mout01.posteo.de (unknown [10.0.0.65])
+	by mout01.posteo.de (Postfix) with ESMTPS id B39111200B9
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+Received: from submission-encrypt01.posteo.de (unknown [10.0.0.75])
+	by mout01.posteo.de (Postfix) with ESMTPS id 8A684160060
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=posteo.de; s=2017;
+	t=1591728262; bh=g3zLYH4xKxcPrHOD18z9YfpQcnk/GaJedfustWU5uGs=;
+	h=To:From:Subject:Date:From;
+	b=brJnt4PLAX3Tda1RHCo91aB1kMAL/Ku9dmO7D2DD41Zu5ShNsyqqyDkyxb1DsDn3O
+	 6KuBZe3/8gemBuCJ/mxzwd9v8sBnlrV+5afIk0Ye9VvthZsc4HoG79+FiVOi9F38o0
+	 DtJJFYFw/X7mAc5Xyt0B0JvtiTPpBdRAkluUQm+QW6cW6GGlwicVW19qvebzq+sHyP
+	 X2bZ8wpo78yVgvjPBK3DLaXa+pKFMBjLdDUcIE2bZnY6u6F1x8SXGKGBoxVwdJipJx
+	 v14so5IejNsf4LYJjH3Qb8xgK1aAi6e6nQn4YXV0INL6ahzgALiT9N6vwunNKYVJNi
+	 fPPKvBWDfUS4Q==
+Received: from customer (localhost [127.0.0.1])
+	by submission (posteo.de) with ESMTPSA id 49hJtt1WPbz6tmV
+	for <hanerthaertidiuea@gmx.de>; Tue,  9 Jun 2020 20:44:22 +0200 (CEST)
+To: hanerthaertidiuea@gmx.de
+From: deltachat <alice@posteo.org>
+Subject: test
+Message-ID: <04422840-f884-3e37-5778-8192fe22d8e1@posteo.de>
+Date: Tue, 9 Jun 2020 20:45:02 +0200
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.8.1
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+Content-Language: de-DE
+Posteo-User: alice@posteo.org
+Posteo-Dkim: ok
+
+--B39111200B9.1591728262/mout01.posteo.de--

test-data/message/testrun_ndn.eml

@@ -0,0 +1,107 @@
+Return-Path: <>
+Delivered-To: alice@testrun.org
+Received: from hq5.merlinux.eu
+	by hq5.merlinux.eu (Dovecot) with LMTP id Ye02K6PB5F43cQAAPzvFDg
+	for <alice@testrun.org>; Sat, 13 Jun 2020 14:08:03 +0200
+Received: by hq5.merlinux.eu (Postfix)
+	id 9EBE627A0B2E; Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+Date: Sat, 13 Jun 2020 14:08:03 +0200 (CEST)
+From: MAILER-DAEMON@hq5.merlinux.eu (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@testrun.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="CDB8D27A0B2C.1592050083/hq5.merlinux.eu"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20200613120803.9EBE627A0B2E@hq5.merlinux.eu>
+
+This is a MIME-encapsulated message.
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host hq5.merlinux.eu.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1
+    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in
+    virtual mailbox table (in reply to RCPT TO command)
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Delivery report
+Content-Type: message/global-delivery-status
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; hq5.merlinux.eu
+X-Postfix-Queue-ID: CDB8D27A0B2C
+X-Postfix-Sender: rfc822; alice@testrun.org
+Arrival-Date: Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+
+Final-Recipient: rfc822; hcksocnsofoejx@five.chat
+Original-Recipient: rfc822;hcksocnsofoejx@five.chat
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; mail.five.chat
+Diagnostic-Code: smtp; 550 5.1.1 <hcksocnsofoejx@five.chat>: Recipient address
+    rejected: User unknown in virtual mailbox table
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu
+Content-Description: Undelivered Message
+Content-Type: message/global
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <alice@testrun.org>
+Received: from localhost (p200300edb723070079835ce22985a199.dip0.t-ipconnect.de [IPv6:2003:ed:b723:700:7983:5ce2:2985:a199])
+	by hq5.merlinux.eu (Postfix) with UTF8SMTPSA id CDB8D27A0B2C;
+	Sat, 13 Jun 2020 14:08:01 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org;
+	s=testrun; t=1592050082;
+	bh=Kvhta0OMsTRVC7OlaAqo68TBE0KuGBv4vUBp6Ez/7VY=;
+	h=Subject:References:In-Reply-To:Date:To:From:From;
+	b=Ql60JEGFXLNvjsyihATw2z34ct++8xZvTPNw0snXe6+oqdqsRZJ9tWNDTxOgx8Iqf
+	 HQ4puBVGcWjIlszYQVLlq3APi04o2ep3GrD8EF0J0GpDdW8yw6wCos6Q8r+TWmXwET
+	 kGXHTRPVaUIqZF2i/utypxMfd1ua0S3jBDnIXTe/p2+XvfC3Cf3hZGW+FQ/Zd7G8Vh
+	 /U2rgX5BTIGf26ZCbmcMaXWkftgv6+yns0AmzorV9yB+EhTkWIUjk+C25bRtMbJ5mZ
+	 93dwdr+sXrrSZLSi+LBqc57Dv9j4p/SUmB4zPlvfUv7/bqLi36pypvtCJ5Ul8UEXSb
+	 XNFZPaEl+mwjA==
+Content-Type: text/plain; charset=utf-8
+Chat-Disposition-Notification-To: alice@testrun.org
+Subject: =?utf-8?q?Message_from_hocuri1=40testrun=2Eorg?=
+MIME-Version: 1.0
+References: <Mr.VSg3KXFUOTG.9sn7JBxZn1W@testrun.org>
+	<Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+In-Reply-To: <Mr.F4nR4LnXb6v.gqVbCJRgsmn@testrun.org>
+Date: Sat, 13 Jun 2020 12:08:01 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@testrun.org; prefer-encrypt=mutual;
+	keydata=xjMEXt3z1xYJKwYBBAHaRw8BAQdAf6MctU/8cmEqwEN9VFZ3gHBFIxKiEaARZl1DFUkI7e
+	rNFTxob2N1cmkxQHRlc3RydW4ub3JnPsKLBBAWCAAzAhkBBQJe3fPXAhsDBAsJCAcGFQgJCgsCAxYC
+	ARYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygAG2IA/1nTmmmkHAc1Bjtx2FOstbaS+N
+	XHjxaK+hkoWllsyhz0AQDJJ1++u7jVZPRn/j1LlByrT3Jv/D1aY14J5rjj+ADVBM44BF7d89cSCisG
+	AQQBl1UBBQEBB0DpSTaZ30dAVwM9PkBe2h+gFyxn9HSorP4XCHJu/lIdPAMBCAfCeAQYFggAIAUCXt
+	3z1wIbDBYhBIZ3ajRUkki89+04sgctAtaIXygAAAoJEActAtaIXygA2QkA/16toWCtseYKw8G1X2j7
+	xYR3Cyabq37hgbesDOThIIzNAP0UCUS8mnunmkS5adEbftRaDi2JZoGxDw46jtJJ2+13Cw==
+Message-ID: <Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org>
+To: <hcksocnsofoejx@five.chat>
+From:  <alice@testrun.org>
+
+F
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+
+--CDB8D27A0B2C.1592050083/hq5.merlinux.eu--

test-data/message/tiscali_ndn.eml

@@ -0,0 +1,91 @@
+Return-Path: <>
+Delivered-To: alice@tiscali.it
+Received: from director-5.mail.tiscali.sys ([10.39.80.174])
+	by dovecot-08.mail.tiscali.sys with LMTP id SBRfEpGb517VAgAAd2fHbg
+	for <alice@tiscali.it>; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from cmgw-4.mail.tiscali.it ([10.39.80.174])
+	by director-5.mail.tiscali.sys with LMTP id MFUPL5Cb516tawAArQJVuQ
+	; Mon, 15 Jun 2020 16:02:25 +0000
+Received: from michael.mail.tiscali.it ([213.205.33.246])
+	by cmgw-4.mail.tiscali.it with 
+	id rTtS2200V5JdeUd01U2RlV; Mon, 15 Jun 2020 16:02:25 +0000
+x-cnfs-analysis: v=2.3 cv=ZdPMyfdA c=1 sm=1 tr=0 cx=a_idp_d
+ a=AfTPebshMYb+aQOCLa9q3Q==:117 a=HpEJnUlJZJkA:10 a=jmdcTMp_Gj4A:10
+ a=r77TgQKjGQsHNAKrUKIA:9 a=b8iBRs35AAAA:8 a=NMdB-582e605uxHDr_AA:9
+ a=QEXdDO2ut3YA:10 a=MhhPCb74-dYA:10 a=HXlsH_Kov2KnitTn7A4A:9
+ a=BkuCPOF3BOzethIN9HQA:9 a=qG5HpJ6ZyD35YNEB:21 a=kvHihYffoorsyJbA:21
+ a=xD8EQi6zkreDqSNPYj5l:22
+Date: Mon, 15 Jun 2020 16:02:25 +0000
+From: Mail Delivery System <mail-daemon@smtp.tiscali.it>
+To: alice@tiscali.it
+Subject: Delivery status notification
+MIME-Version: 1.0
+Content-Type: multipart/report; boundary="------------I305M09060309060P_896715922369450"
+
+This is a multi-part message in MIME format.
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/plain; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+      This is an automatically generated Delivery Status Notification.      
+
+Delivery to the following recipients was aborted after 2 second(s):
+
+  * shenauithz@testrun.org
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: message/delivery-status; charset=UTF-8;
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; michael.mail.tiscali.it [213.205.33.13]
+Received-From-MTA: dns; localhost [146.241.100.150]
+Arrival-Date: Mon, 15 Jun 2020 16:02:23 +0000
+
+
+Final-recipient: rfc822; shenauithz@testrun.org
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550 5.1.1 <shenauithz@testrun.org>: Recipient address rejected: User unknown in virtual mailbox table
+Last-attempt-Date: Mon, 15 Jun 2020 16:02:25 +0000
+
+
+
+--------------I305M09060309060P_896715922369450
+Content-Type: text/rfc822-headers; Content-Transfer-Encoding: 8bit
+Content-Disposition: attachment
+
+x-auth-user: alice@tiscali.it
+Chat-Disposition-Notification-To: alice@tiscali.it
+Chat-User-Avatar: avatar.jpg
+Subject: =?utf-8?q?Message_from_=F0=9F=8F=9E=EF=B8=8F_Mefiscali?=
+MIME-Version: 1.0
+Date: Mon, 15 Jun 2020 16:02:22 +0000
+X-Mailer: Delta Chat Core 1.35.0/Android 1.9.5
+Chat-Version: 1.0
+Autocrypt: addr=alice@tiscali.it; prefer-encrypt=mutual;
+	keydata=xjMEXtFRUBYJKwYBBAHaRw8BAQdA5sqHJqkWlveCgsNd0rtwtZrT1mmo1gwaGC5+WheYk5
+	nNHTxhbmRyZWFzLmxhdHRtYW5uQHRpc2NhbGkuaXQ+wosEEBYIADMCGQEFAl7RUXoCGwMECwkIBwYV
+	CAkKCwIDFgIBFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrAAaAEA4wssXeU2IXnowv
+	iu3zmcNzDgE4HdmW4RFyqJC6bgxXQA/3aTfE/PhQgZvi6RrKMvP4zygXpD9y+3ydIZP88Bp8kIzjgE
+	XtFRUBIKKwYBBAGXVQEFAQEHQKaAwlP0j9m0aYsCtO+qD9+foH0kiTN5BWDe5YcZrckVAwEIB8J4BB
+	gWCAAgBQJe0VF6AhsMFiEEJUsbRIjZEaRNAs1p1Z5M1vshkrAACgkQ1Z5M1vshkrA1JgEAkscCQlps
+	h3ZxlLqBlbf2+85f4S4aGQfFPtIYEkKKhYEBAJbQulNNp9UarvhfyBiIdvkBVDcCnJZwzbORqp8RM0 gC
+Message-ID: <Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it>
+To: <shenauithz@testrun.org>
+From: =?utf-8?b?8J+Pnu+4jyBNZWZpc2NhbGk=?= <alice@tiscali.it>
+Content-Type: multipart/mixed; boundary="5uAmYQux1HZxxriijTjjKSp4DMoJwq"
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tiscali.it; s=smtp;
+	t=1592236943; bh=C3taz+zuSre1ko5Q5CzGPmbyrgegKYBClx/3Dv7t/Xw=;
+	h=Subject:Date:To:From;
+	b=LrcfLfrQoemOkHTQsqR8MExqNlx5KPYNFWhwlBWylvVc5GlmlhzqM6SAVKd0NVsKE
+	 gVRlBId5FvnlwoJ2WZnXaw/+3lWKilMTuzzQ1oFGvLnZ1XUaUEfuliIv+9NI79dJWX
+	 +S3jsSgzJMJc9+fO6s9bJsX1EHQ2a8GXNbwDtLXs=
+
+
+
+--------------I305M09060309060P_896715922369450--
+
+

test-data/message/videochat_invitation.eml

@@ -0,0 +1,15 @@
+Content-Type: text/plain; charset=utf-8
+Subject: Message from user
+Message-ID: <Mr.f1O61111evx.ikocf333353@example.org>
+Date: Mon, 20 Jul 2020 14:28:30 +0000
+X-Mailer: Delta Chat Core 1.40.0/CLI
+Chat-Version: 1.0
+Chat-Content: videochat-invitation
+Chat-Webrtc-Room: https://example.org/p2p/?roomname=6HiduoAn4xN
+To: <tunis3@example.org>
+From: "=?utf-8?q??=" <tunis4@example.org>
+
+You are invited to an videochat, click https://example.org/p2p/?roomname=6HiduoAn4xN to join.
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat

test-data/message/yahoo_ndn.eml

@@ -0,0 +1,100 @@
+X-Atlas-Received: from 10.218.250.153 by atlas222.free.mail.ne1.yahoo.com with http; Wed, 10 Jun 2020 10:11:23 +0000
+Return-Path: <>
+Received: from 77.238.177.145 (EHLO sonic314-19.consmr.mail.ir2.yahoo.com)
+ by atlas222.free.mail.ne1.yahoo.com with SMTPs; Wed, 10 Jun 2020 10:11:23 +0000
+X-Originating-Ip: [77.238.177.145]
+Received-SPF: none (domain of sonic314-19.consmr.mail.ir2.yahoo.com does not designate permitted sender hosts)
+Authentication-Results: atlas222.free.mail.ne1.yahoo.com;
+ dkim=pass header.i=@yahoo.com header.s=@bounce;
+ spf=none smtp.mailfrom=sonic314-19.consmr.mail.ir2.yahoo.com;
+ dmarc=success(p=REJECT) header.from=yahoo.com;
+X-Apparently-To: alice@yahoo.com; Wed, 10 Jun 2020 10:11:23 +0000
+X-YMailISG: RT5ZnycWLDvIW52uqHS_EWNgl31NdJPyLLB2F4SYb1GCAoo9
+ pcninuVU5GDMBZykeMT4cSUt4ZqXxS5FdEeWJqtGIAtbEGbIL8Uhcoszqm4m
+ JuMJiQZwEE7W_fsS_9MUK5gZtMkhKkSnAuaeaOLKNYAwFZdBqA0uEYA5EmVf
+ EC9J4RGQ4hZvrMqMj_W.cj4pvbEC.pyirLxTfkICuUkZVguYoxG16y1EOJPw
+ B48fhXvF5ErU7WAHKxyRM3bMOg7b5pXHKn1dtRSVAXEuqBAQrWig1pePpYH1
+ wO54sYT7cgmdiFvfLY5rR7YcBzopmKJBycKzBVoRLCY4gvoNyTLPKx9o3AAz
+ WU4B7TGejDBElYSLpfnyvQg8wU27zzo2IVBZWUNztP0Ca8CQ07Y7TxUZAO.f
+ DNO5c7nd81PHMRDbSeaw1BTV2Yd9vlBc7syYmwGvtVBJQwRU7qPN.DpFO2jC
+ 9j9DytVhm5231gdBBRSzW78yG.VvaIdJgq_YViKNM9VxFseTz3Sjt3TaYznP
+ gAVq.MxpopNsSZf_tedwAhXDWyrjKsRPK.v2ANivmuWGPednniEaMYhxJ05M
+ _5SnJ.hAU.l6h3HCEfiU.SH390_3tZgYNfxCo4GPPFMfnNPmKa3.rgpChBCz
+ 9CRexJ8BSFyCEeAhuqQ8vSJfSuittJmXvS6Tk8Rxd9HUJAtKzZ.xCWZQ4tA6
+ Yp2aRG23_rK_C6hH8ArkkvbG.uVQTt6DltSX6avJLObBfIhBH0x64RoFjGee
+ vYXxM741Okm0jH7r79c8GhnAwas_bwfkaTW9e1nhYP0eyI36z_QwLYgOH3Mm
+ LrUcejpOMDR60QWDuDyRbWXOJdr3Q2K0ERhuAy6YnINq0sL3HX7t5wjsFLvp
+ _7Ri_eruTfIst4C7DZwERwui6aDSEAdF1Z8oZukBVmiyZsHmhJQCUik646iy
+ 3ASMR3lX7R3q2PBHQo2oC3qte8Fzz1FhKoMtfCGtIpeCazlkhDEJ6eTBSQ3R
+ Pe7M_GPiv3QNp7qu5CWHlzy6hWEKIkNwx.WRGYzfxkyJMmJm4UrhQYUfa4lG
+ Wb8n.mfYnS_KGYtzyRFNqAL0IGo.1MB9aG6qQk456Fz9GJgbHLWrMXVtyfrr
+ Uo7mKih8FCrdUKv5X6KBnpY0vvyoH5jrWyrvo3DW0bq_JvZ9U51JwUhoGY5U
+ c1t.yCSJbs8tnrGZHuUTOvouWzpCAJsk34AqRyH0wDJZQsAwBW5UZ3jx8ARA
+ FicoSqZCa4wEP9WaaXvfzFbmLW0-
+X-Originating-IP: [77.238.177.145]
+Received: from 10.217.135.165  (EHLO sonic314-19.consmr.mail.ir2.yahoo.com) (77.238.177.145)
+  by mta4277.mail.ne1.yahoo.com with SMTPS; Wed, 10 Jun 2020 10:11:21 +0000
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=bounce; t=1591783879; bh=4BmZzBC/nu0AJ9r0i0xNuCENks2KZcuXCbjSHdzbg9Y=; h=Date:From:To:Subject:From:Subject; b=lPxu8goOGOLVgnwbndfdptZ7zI5VEo0lSSr+ONGxwdtuhrySKDU6Sp41/g6jWbAiVPT1947j/B5wOlPfa5tv4XkWrGf0JCbT1I20ZJIkNfNwt4F0qPnbJAiHFIDPxcY68utjC9IgPWJd0cGqJNXbFwbJBu88rtrbMoInzLakh5I=
+Received: from sonic.gate.mail.ne1.yahoo.com by sonic314.consmr.mail.ir2.yahoo.com with HTTP; Wed, 10 Jun 2020 10:11:19 +0000
+Date: Wed, 10 Jun 2020 10:11:19 +0000
+From: MAILER-DAEMON@yahoo.com
+To: alice@yahoo.com
+Message-ID: <1713051795.39992.1591783879940@sonic314.consmr.mail.ir2.yahoo.com>
+Subject: Failure Notice
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+Content-Length: 3347
+
+Sorry, we were unable to deliver your message to the following address.
+
+<haeclirth.sinoenrat@yahoo.com>:
+554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com
+
+--- Below this line is a copy of the message.
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=s2048; t=1591783876; bh=kXD6TZuQDjqZf/AEAJ2HCX32Titkj3IywytG6GSm4yQ=; h=Date:From:To:Subject:References:From:Subject; b=PexnIBxnVSkyutv/jVn+Wlt5QPuVHnWleP3baWlvqkXaKR51pCaZIoGJggMEonitEeJkcYpgplBawEBz0hlGf63NqOHHkxUk6U1olwc0y9kj5kDH5lrORgXrf7U4z5t+i5n0II36MxbG9n/5tDRXoiabLFoWx//3O9x/ZJvVWPlq8RBFzVG8aoL2TkBQAVcqX/vW1f3WJuopaUYWB4AzR3TyuC2kVQPFqbPMk+G/VmyuZFmZesh1bSBva5hdKYLxES5v8hvTIDRqSYZrnZ4V67MqicZ8m229Xf4Za8qOE+a2Z+Vv5VrQ+CjjPZwRAcmKHkLY80VCSkpeL2R2YG4APg==
+X-YMail-OSG: 4spnw98VM1k_g51CM6oepdNEMiPFRtZ0ZG_zOBGIlhcNvS1mkzr8l8VTB0CY_Q0
+ dR52ikl7QVYESerRQgcGqBfLwKpem7i1XSrCl2HuyHeWzF6Gu5MqFPMCak.v8GyDXNO075NEwNt1
+ i18CJ29cEjiHthoamgmj0oqerAglglKRhTuuAFy4wUmZZm7VyvaW4wHUD1g7DeWGijQsCglSYMUK
+ CmoFcKsWOZBSYPMkp7iRwUp52pXHFin3qf4uQ27K_Sh.6s7KLAfWVkV7L_5AR3MyCPAVzm71.1yG
+ G7Vy5HSBgGMQ90B7VbcjOkCg3F4JNl4Z_P2ejV1KZ.tNoPLgO.FmsfFy1OXBGf3m2mDmRcuEO4K2
+ mTRhsjZf.2iiWpx02b3tY.oUtYrIBXBVIFPbTB9sBMn_9Z_qdVmO3gjD6gCPEBzuVvEO0eZIrgaw
+ EDTZt8Z9tSRDm1.4gV8LWQBYShF7XuMV0togiLYIO8s_iTHcTbhKhPlwxP.mxr06xcx_9kzReVTL
+ 9lB1FkB5Jm0WccWHGhLBqeMjGDoaNqPxLqJ.1tI58tLXsPoR6m1NFVEdzI1G.4AVBeXZ_9BjgUhm
+ KY33sEg.GwIjUlWWWuSyRZ1q1K2nqi1z29wH2R1Glmdmx0lyqfMg9Xe8HV7YZu2CuZ8SlDLLB.rX
+ NU4PwMsNfU6pK2HejQPsJuyOlI5Q824rXRF5xTLYKsYcQptoFXyLe6MXKW1ThBLQV2nWYDRs_V.e
+ SBmt22TfuOwu4Y5ju0sXmztZ8zpiIC8_rnAa5bVBEHxzkic64UZdukDX9V12Pk3G2sGYRyPTH472
+ wBX33JpDuq6BtrKr4FXjCLeVppARTHpiKM0jHMjmNf1bF0TvrcCsC9zAYtitAqgcGZNFETNuV3KM
+ 57XifdDEwUPOuww0ApSWO.iwP2POvIRBVlrxdgA8MbLmuuX4UxNCw23z1f7MVY6F3L60LUrX5GZO
+ aKaMmD1XTzx32J6c_TUmyuViT5vphqpEooTzHG2X7ALb4xC8yHlE4wDKyaEDARZ.8P2lO9T18oCz
+ OQvJjwDaLOkeAmo23yRMn70bYJK3tP9Z5cS1C0TE8PEtz4sd1syQUIZZ2g8JG_AQcE4lUZSZlIKN
+ AHjB8h8Uin35zKe0Le1DBjdQUmpgAETlmYE7V0nJDEmagB3dtpbokgRBuuBfhXlFpxHcnAmBFFFm
+ XOSLWEPnmxu2o8CCjjz3QUBy2fr3EI_D2VFpy..MuZgRwtES.l24m_95xtQxI28R4SWZN6LsS_rr
+ 1S33BJCCCAfXtCAFzCfz5.qSzHRYbLdY5do6yKj0pPLQTUTjlMwmCUGPcSJhsyxkkEVIK1W_Z16R
+ ZRls-
+Received: from sonic.gate.mail.ne1.yahoo.com by sonic314.consmr.mail.ir2.yahoo.com with HTTP; Wed, 10 Jun 2020 10:11:16 +0000
+Date: Wed, 10 Jun 2020 10:11:12 +0000 (UTC)
+From: Delta Chat Test <alice@yahoo.com>
+To: "haeclirth.sinoenrat@yahoo.com" <haeclirth.sinoenrat@yahoo.com>
+Message-ID: <1680295672.3657931.1591783872936@mail.yahoo.com>
+Subject: test
+MIME-Version: 1.0
+Content-Type: multipart/alternative; 
+	boundary="----=_Part_3657930_145367320.1591783872935"
+References: <1680295672.3657931.1591783872936.ref@mail.yahoo.com>
+X-Mailer: WebService/1.1.16072 YMailNorrin Mozilla/5.0 (X11; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0
+Content-Length: 494
+
+------=_Part_3657930_145367320.1591783872935
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+test
+
+------=_Part_3657930_145367320.1591783872935
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+<html><head></head><body><div class="yahoo-style-wrap" style="font-family:Helvetica Neue, Helvetica, Arial, sans-serif;font-size:16px;"><div dir="ltr" data-setdir="false">test<br></div></div></body></html>
+------=_Part_3657930_145367320.1591783872935--

tests/stress.rs

@@ -99,9 +99,14 @@ struct TestContext {
 }
 
 async fn create_test_context() -> TestContext {
+    use rand::Rng;
+
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    let ctx = Context::new("FakeOs".into(), dbfile.into()).await.unwrap();
+    let id = rand::thread_rng().gen();
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), id)
+        .await
+        .unwrap();
     TestContext { ctx, dir }
 }