bump version to 1.45

Update async-imap to fix Oauth2

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.43.1
+          toolchain: 1.45.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,21 +32,34 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.43.1
+            toolchain: 1.45.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all
 
 
   build_and_test:
     name: Build and test
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly, 1.43.1]
+        rust: [1.45.0]
+        experimental: [false]
+#        include:
+#          - os: ubuntu-latest
+#            rust: nightly
+#            experimental: true
+#          - os: windows-latest
+#            rust: nightly
+#            experimental: true
+#          - os: macOS-latest
+#            rust: nightly
+#            experimental: true
 
     steps:
     - uses: actions/checkout@master
@@ -79,11 +92,10 @@ jobs:
       uses: actions-rs/cargo@v1
       with:
         command:  check
-        args: --workspace --all --bins --examples --tests
+        args: --all --bins --examples --tests --features repl
 
     - name: tests
       uses: actions-rs/cargo@v1
       with:
         command: test
-        args: --workspace
-          
+        args: --all

ASYNC-API-TODO.txt

@@ -1,54 +0,0 @@
-
-Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
-
-- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
-
-APIs:
-    dc_context_new                 # opens the database
-    dc_open                             # FFI only
-      -> drop it and move parameters to dc_context_new()
-
-    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
-    dc_imex NEVER goes through the job system
-    dc_imex import_backup needs to ensure dc_stop_jobs()
-
-    dc_start_io                # start smtp/imap and job handling subsystems
-    dc_stop_io                 # stop smtp/imap and job handling subsystems
-    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
-
-    dc_close                            # FFI only
-      -> can be dropped
-    dc_context_unref
-
-    for ios share-extension:
-    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
-        0: message was sent
-        1: message failed to go out, is queued as a job to be retried later
-        2: message permanently failed?
-
-EVENT handling:
-    start a callback thread and call get_next_event() which is BLOCKING
-    it's fine to start this callback thread later, it will see all events.
-    Note that the core infinitely fills the internal queue if you never drain it.
-
-    FFI-get_next_event() returns NULL if the context is unrefed already?
-
-   sidenote: how python's callback thread does it currently:
-         CB-thread runs this while loop:
-         while not QUITFLAG:
-             ev = context.get_next_event( )
-             ...
-         So in order to shutdown properly one has to set QUITFLAG
-         before calling dc_stop_jobs() and dc_context_unref
-
-    event API:
-        get_data1_int
-        get_data2_int
-        get_data3_str
-
-
-- userdata likely only used for the callbacks, likely can be dropped, needs verification
-
-
-- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
-

CHANGELOG.md

@@ -1,5 +1,126 @@
 # Changelog 
 
+## 1.45.0
+
+- add `dc_accounts_t` account manager object and related api functions #1784
+
+- add capability to import backups as .tar files,
+  which will become the default in a subsequent release #1749
+
+- try various server domains on configuration #1780 #1838
+
+- recognize .tgs files as stickers #1826
+
+- remove X-Mailer debug header #1819
+
+- improve guessing message types from extension #1818
+
+- fix showing unprotected subjects in encrypted messages #1822
+
+- fix threading in interaction with non-delta-clients #1843
+
+- fix handling if encryption degrades #1829
+
+- fix webrtc-servers names set by the user #1831
+
+- update provider database #1828
+
+- update async-imap to fix Oauth2 #1837
+
+- optimize jpeg assets with trimage #1840
+
+- add tests and documentations #1809 #1820
+
+
+## 1.44.0
+
+- fix peerstate issues #1800 #1805
+
+- fix a crash related to muted chats #1803
+
+- fix incorrect dimensions sometimes reported for images #1806
+
+- fixed `dc_chat_get_remaining_mute_duration` function #1807
+
+- handle empty tags (e.g. `<br/>`) in HTML mails #1810
+
+- always translate the message about disappearing messages timer change #1813
+
+- improve footer detection in plain text email #1812
+
+- update device chat icon to fix warnings in iOS logs #1802
+
+- fix deletion of multiple messages #1795
+
+
+## 1.43.0
+
+- improve using own jitsi-servers #1785
+
+- fix smtp-timeout tweaks for larger mails #1797
+
+- more bug fixes and updates #1794 #1792 #1789 #1787
+
+
+## 1.42.0
+
+- new qr-code type `DC_QR_WEBRTC` #1779
+
+- new `dc_chatlist_get_summary2()` api #1771
+
+- tweak smtp-timeout for larger mails #1782
+
+- optimize read-receipts #1765
+
+- Allow http scheme for DCACCOUNT URLs #1770
+
+- improve tests #1769
+
+- bug fixes #1766 #1772 #1773 #1775 #1776 #1777
+
+
+## 1.41.0
+
+- new apis to initiate video chats #1718 #1735
+
+- new apis `dc_msg_get_ephemeral_timer()`
+  and `dc_msg_get_ephemeral_timestamp()`
+
+- new api `dc_chatlist_get_summary2()` #1771
+
+- improve IMAP handling #1703 #1704
+
+- improve ephemeral messages #1696 #1705
+
+- mark location-messages as auto-generated #1715
+
+- multi-device avatar-sync #1716 #1717
+
+- improve python bindings #1732 #1733 #1738 #1769
+
+- Allow http scheme for DCACCOUNT urls #1770
+
+- more fixes #1702 #1706 #1707 #1710 #1719 #1721
+  #1723 #1734 #1740 #1744 #1748 #1760 #1766 #1773 #1765
+
+- refactorings #1712 #1714 #1757
+
+- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
+
+
+## 1.40.0
+
+- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
+
+- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
+
+- improve idle #1690 #1688
+
+- fix message processing issues by sequential processing #1694
+
+- refactorings #1670 #1673
+
+
 ## 1.39.0
 
 - fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.39.0"
+version = "1.45.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -20,10 +20,10 @@ smallvec = "1.0.0"
 surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { version = "0.3" }
+async-smtp = "0.3"
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.3.1"
+async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
 async-std = { version = "1.6.1", features = ["unstable"] }
 base64 = "0.12"
@@ -43,8 +43,7 @@ strum = "0.18.0"
 strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
-itertools = "0.8.0"
-image-meta = "0.1.0"
+itertools = "0.9.0"
 quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
@@ -60,11 +59,15 @@ anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
 async-std-resolver = "0.19.5"
+async-tar = "0.3.0"
+uuid = { version = "0.8", features = ["serde", "v4"] }
 
 pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+dirs = { version = "3.0.1", optional=true }
+toml = "0.5.6"
 
 
 [dev-dependencies]
@@ -95,7 +98,7 @@ required-features = ["repl"]
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 

README.md

@@ -95,7 +95,8 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-
+- `RUST_LOG=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+SMTP tracing in addition to info messages.
 
 ### Expensive tests
 

ci_scripts/docker-coredeps/Dockerfile

@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust nightly
+# Install Rust
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share

ci_scripts/run_all.sh

@@ -15,6 +15,7 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
+export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.39.0"
+version = "1.45.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -23,6 +23,7 @@ serde_json = "1.0"
 async-std = "1.6.0"
 anyhow = "1.0.28"
 thiserror = "1.0.14"
+rand = "0.7.3"
 
 [features]
 default = ["vendored"]

deltachat-ffi/build.rs

@@ -19,10 +19,10 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
+        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();

deltachat-ffi/src/dc_array.rs

@@ -1,46 +1,56 @@
+use crate::chat::ChatItem;
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
+use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
+    MsgIds(Vec<MsgId>),
+    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub fn new(capacity: usize) -> Self {
-        dc_array_t::Uint(Vec::with_capacity(capacity))
-    }
-
-    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
-    pub fn new_locations(capacity: usize) -> Self {
-        dc_array_t::Locations(Vec::with_capacity(capacity))
-    }
-
-    pub fn add_id(&mut self, item: u32) {
-        if let Self::Uint(array) = self {
-            array.push(item);
-        } else {
-            panic!("Attempt to add id to array of other type");
-        }
-    }
-
-    pub fn add_location(&mut self, location: Location) {
-        if let Self::Locations(array) = self {
-            array.push(location)
-        } else {
-            panic!("Attempt to add a location to array of other type");
-        }
-    }
-
-    pub fn get_id(&self, index: usize) -> u32 {
+    pub(crate) fn get_id(&self, index: usize) -> u32 {
         match self {
+            Self::MsgIds(array) => array[index].to_u32(),
+            Self::Chat(array) => match array[index] {
+                ChatItem::Message { msg_id } => msg_id.to_u32(),
+                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
+                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
+            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index] as u32,
+            Self::Uint(array) => array[index],
         }
     }
 
-    pub fn get_location(&self, index: usize) -> &Location {
+    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(array) => array.get(index).and_then(|item| match item {
+                ChatItem::Message { .. } => None,
+                ChatItem::Marker1 { .. } => None,
+                ChatItem::DayMarker { timestamp } => Some(*timestamp),
+            }),
+            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(_) => None,
+            Self::Locations(array) => array
+                .get(index)
+                .and_then(|location| location.marker.as_deref()),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -48,55 +58,18 @@ impl dc_array_t {
         }
     }
 
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Locations(array) => array.is_empty(),
-            Self::Uint(array) => array.is_empty(),
-        }
-    }
-
     /// Returns the number of elements in the array.
-    pub fn len(&self) -> usize {
+    pub(crate) fn len(&self) -> usize {
         match self {
+            Self::MsgIds(array) => array.len(),
+            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub fn clear(&mut self) {
-        match self {
-            Self::Locations(array) => array.clear(),
-            Self::Uint(array) => array.clear(),
-        }
-    }
-
-    pub fn search_id(&self, needle: u32) -> Option<usize> {
-        if let Self::Uint(array) = self {
-            for (i, &u) in array.iter().enumerate() {
-                if u == needle {
-                    return Some(i);
-                }
-            }
-            None
-        } else {
-            panic!("Attempt to search for id in array of other type");
-        }
-    }
-
-    pub fn sort_ids(&mut self) {
-        if let dc_array_t::Uint(v) = self {
-            v.sort();
-        } else {
-            panic!("Attempt to sort array of something other than uints");
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const u32 {
-        if let dc_array_t::Uint(v) = self {
-            v.as_ptr()
-        } else {
-            panic!("Attempt to convert array of something other than uints to raw");
-        }
+    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
+        (0..self.len()).find(|i| self.get_id(*i) == needle)
     }
 }
 
@@ -106,6 +79,18 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
+impl From<Vec<MsgId>> for dc_array_t {
+    fn from(array: Vec<MsgId>) -> Self {
+        dc_array_t::MsgIds(array)
+    }
+}
+
+impl From<Vec<ChatItem>> for dc_array_t {
+    fn from(array: Vec<ChatItem>) -> Self {
+        dc_array_t::Chat(array)
+    }
+}
+
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -118,12 +103,11 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let mut arr = dc_array_t::new(7);
-        assert!(arr.is_empty());
+        let arr: dc_array_t = Vec::<u32>::new().into();
+        assert!(arr.len() == 0);
 
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
 
         assert_eq!(arr.len(), 1000);
 
@@ -131,31 +115,15 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        arr.clear();
-
-        assert!(arr.is_empty());
-
-        arr.add_id(13);
-        arr.add_id(7);
-        arr.add_id(666);
-        arr.add_id(0);
-        arr.add_id(5000);
-
-        arr.sort_ids();
-
-        assert_eq!(arr.get_id(0), 0);
-        assert_eq!(arr.get_id(1), 7);
-        assert_eq!(arr.get_id(2), 13);
-        assert_eq!(arr.get_id(3), 666);
+        assert_eq!(arr.search_id(10), Some(8));
+        assert_eq!(arr.search_id(1), None);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let mut arr = dc_array_t::new(7);
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/lib.rs

@@ -1,3 +1,4 @@
+#![deny(clippy::all)]
 #![allow(
     non_camel_case_types,
     non_snake_case,
@@ -23,10 +24,12 @@ use std::time::{Duration, SystemTime};
 use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
+use deltachat::accounts::Accounts;
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
+use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock::StockMessage;
@@ -36,6 +39,7 @@ mod dc_array;
 
 mod string;
 use self::string::*;
+use deltachat::chatlist::Chatlist;
 
 // as C lacks a good and portable error handling,
 // in general, the C Interface is forgiving wrt to bad parameters.
@@ -72,13 +76,17 @@ pub unsafe extern "C" fn dc_context_new(
     };
 
     let ctx = if blobdir.is_null() || *blobdir == 0 {
-        block_on(Context::new(os_name, as_path(dbfile).to_path_buf().into()))
-    } else {
-        block_on(Context::with_blobdir(
+        use rand::Rng;
+        // generate random ID as this functionality is not yet available on the C-api.
+        let id = rand::thread_rng().gen();
+        block_on(Context::new(
             os_name,
             as_path(dbfile).to_path_buf().into(),
-            as_path(blobdir).to_path_buf().into(),
+            id,
         ))
+    } else {
+        eprintln!("blobdir can not be defined explicitly anymore");
+        return ptr::null_mut();
     };
     match ctx {
         Ok(ctx) => Box::into_raw(Box::new(ctx)),
@@ -298,6 +306,16 @@ pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c
     block_on(ctx.is_io_running()) as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
+    if context.is_null() {
+        return 0;
+    }
+    let ctx = &*context;
+
+    ctx.get_id() as libc::c_int
+}
+
 #[no_mangle]
 pub type dc_event_t = Event;
 
@@ -329,37 +347,38 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         return 0;
     }
 
-    let event = &*event;
+    let event = &(*event).typ;
     match event {
-        Event::Info(_)
-        | Event::SmtpConnected(_)
-        | Event::ImapConnected(_)
-        | Event::SmtpMessageSent(_)
-        | Event::ImapMessageDeleted(_)
-        | Event::ImapMessageMoved(_)
-        | Event::ImapFolderEmptied(_)
-        | Event::NewBlobFile(_)
-        | Event::DeletedBlobFile(_)
-        | Event::Warning(_)
-        | Event::Error(_)
-        | Event::ErrorNetwork(_)
-        | Event::ErrorSelfNotInGroup(_) => 0,
-        Event::MsgsChanged { chat_id, .. }
-        | Event::IncomingMsg { chat_id, .. }
-        | Event::MsgDelivered { chat_id, .. }
-        | Event::MsgFailed { chat_id, .. }
-        | Event::MsgRead { chat_id, .. }
-        | Event::ChatModified(chat_id) => chat_id.to_u32() as libc::c_int,
-        Event::ContactsChanged(id) | Event::LocationChanged(id) => {
+        EventType::Info(_)
+        | EventType::SmtpConnected(_)
+        | EventType::ImapConnected(_)
+        | EventType::SmtpMessageSent(_)
+        | EventType::ImapMessageDeleted(_)
+        | EventType::ImapMessageMoved(_)
+        | EventType::ImapFolderEmptied(_)
+        | EventType::NewBlobFile(_)
+        | EventType::DeletedBlobFile(_)
+        | EventType::Warning(_)
+        | EventType::Error(_)
+        | EventType::ErrorNetwork(_)
+        | EventType::ErrorSelfNotInGroup(_) => 0,
+        EventType::MsgsChanged { chat_id, .. }
+        | EventType::IncomingMsg { chat_id, .. }
+        | EventType::MsgDelivered { chat_id, .. }
+        | EventType::MsgFailed { chat_id, .. }
+        | EventType::MsgRead { chat_id, .. }
+        | EventType::ChatModified(chat_id)
+        | EventType::ChatEphemeralTimerModified { chat_id, .. } => chat_id.to_u32() as libc::c_int,
+        EventType::ContactsChanged(id) | EventType::LocationChanged(id) => {
             let id = id.unwrap_or_default();
             id as libc::c_int
         }
-        Event::ConfigureProgress(progress) | Event::ImexProgress(progress) => {
+        EventType::ConfigureProgress(progress) | EventType::ImexProgress(progress) => {
             *progress as libc::c_int
         }
-        Event::ImexFileWritten(_) => 0,
-        Event::SecurejoinInviterProgress { contact_id, .. }
-        | Event::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
+        EventType::ImexFileWritten(_) => 0,
+        EventType::SecurejoinInviterProgress { contact_id, .. }
+        | EventType::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
     }
 }
 
@@ -370,35 +389,36 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         return 0;
     }
 
-    let event = &*event;
+    let event = &(*event).typ;
 
     match event {
-        Event::Info(_)
-        | Event::SmtpConnected(_)
-        | Event::ImapConnected(_)
-        | Event::SmtpMessageSent(_)
-        | Event::ImapMessageDeleted(_)
-        | Event::ImapMessageMoved(_)
-        | Event::ImapFolderEmptied(_)
-        | Event::NewBlobFile(_)
-        | Event::DeletedBlobFile(_)
-        | Event::Warning(_)
-        | Event::Error(_)
-        | Event::ErrorNetwork(_)
-        | Event::ErrorSelfNotInGroup(_)
-        | Event::ContactsChanged(_)
-        | Event::LocationChanged(_)
-        | Event::ConfigureProgress(_)
-        | Event::ImexProgress(_)
-        | Event::ImexFileWritten(_)
-        | Event::ChatModified(_) => 0,
-        Event::MsgsChanged { msg_id, .. }
-        | Event::IncomingMsg { msg_id, .. }
-        | Event::MsgDelivered { msg_id, .. }
-        | Event::MsgFailed { msg_id, .. }
-        | Event::MsgRead { msg_id, .. } => msg_id.to_u32() as libc::c_int,
-        Event::SecurejoinInviterProgress { progress, .. }
-        | Event::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
+        EventType::Info(_)
+        | EventType::SmtpConnected(_)
+        | EventType::ImapConnected(_)
+        | EventType::SmtpMessageSent(_)
+        | EventType::ImapMessageDeleted(_)
+        | EventType::ImapMessageMoved(_)
+        | EventType::ImapFolderEmptied(_)
+        | EventType::NewBlobFile(_)
+        | EventType::DeletedBlobFile(_)
+        | EventType::Warning(_)
+        | EventType::Error(_)
+        | EventType::ErrorNetwork(_)
+        | EventType::ErrorSelfNotInGroup(_)
+        | EventType::ContactsChanged(_)
+        | EventType::LocationChanged(_)
+        | EventType::ConfigureProgress(_)
+        | EventType::ImexProgress(_)
+        | EventType::ImexFileWritten(_)
+        | EventType::ChatModified(_) => 0,
+        EventType::MsgsChanged { msg_id, .. }
+        | EventType::IncomingMsg { msg_id, .. }
+        | EventType::MsgDelivered { msg_id, .. }
+        | EventType::MsgFailed { msg_id, .. }
+        | EventType::MsgRead { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::SecurejoinInviterProgress { progress, .. }
+        | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
+        EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
     }
 }
 
@@ -409,44 +429,55 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         return ptr::null_mut();
     }
 
-    let event = &*event;
+    let event = &(*event).typ;
 
     match event {
-        Event::Info(msg)
-        | Event::SmtpConnected(msg)
-        | Event::ImapConnected(msg)
-        | Event::SmtpMessageSent(msg)
-        | Event::ImapMessageDeleted(msg)
-        | Event::ImapMessageMoved(msg)
-        | Event::ImapFolderEmptied(msg)
-        | Event::NewBlobFile(msg)
-        | Event::DeletedBlobFile(msg)
-        | Event::Warning(msg)
-        | Event::Error(msg)
-        | Event::ErrorNetwork(msg)
-        | Event::ErrorSelfNotInGroup(msg) => {
+        EventType::Info(msg)
+        | EventType::SmtpConnected(msg)
+        | EventType::ImapConnected(msg)
+        | EventType::SmtpMessageSent(msg)
+        | EventType::ImapMessageDeleted(msg)
+        | EventType::ImapMessageMoved(msg)
+        | EventType::ImapFolderEmptied(msg)
+        | EventType::NewBlobFile(msg)
+        | EventType::DeletedBlobFile(msg)
+        | EventType::Warning(msg)
+        | EventType::Error(msg)
+        | EventType::ErrorNetwork(msg)
+        | EventType::ErrorSelfNotInGroup(msg) => {
             let data2 = msg.to_c_string().unwrap_or_default();
             data2.into_raw()
         }
-        Event::MsgsChanged { .. }
-        | Event::IncomingMsg { .. }
-        | Event::MsgDelivered { .. }
-        | Event::MsgFailed { .. }
-        | Event::MsgRead { .. }
-        | Event::ChatModified(_)
-        | Event::ContactsChanged(_)
-        | Event::LocationChanged(_)
-        | Event::ConfigureProgress(_)
-        | Event::ImexProgress(_)
-        | Event::SecurejoinInviterProgress { .. }
-        | Event::SecurejoinJoinerProgress { .. } => ptr::null_mut(),
-        Event::ImexFileWritten(file) => {
+        EventType::MsgsChanged { .. }
+        | EventType::IncomingMsg { .. }
+        | EventType::MsgDelivered { .. }
+        | EventType::MsgFailed { .. }
+        | EventType::MsgRead { .. }
+        | EventType::ChatModified(_)
+        | EventType::ContactsChanged(_)
+        | EventType::LocationChanged(_)
+        | EventType::ConfigureProgress(_)
+        | EventType::ImexProgress(_)
+        | EventType::SecurejoinInviterProgress { .. }
+        | EventType::SecurejoinJoinerProgress { .. }
+        | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
+        EventType::ImexFileWritten(file) => {
             let data2 = file.to_c_string().unwrap_or_default();
             data2.into_raw()
         }
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_event_get_account_id(event: *mut dc_event_t) -> u32 {
+    if event.is_null() {
+        eprintln!("ignoring careless call to dc_event_get_account_id()");
+        return 0;
+    }
+
+    (*event).id
+}
+
 #[no_mangle]
 pub type dc_event_emitter_t = EventEmitter;
 
@@ -482,7 +513,7 @@ pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *
     events
         .recv_sync()
         .map(|ev| Box::into_raw(Box::new(ev)))
-        .unwrap_or_else(|| ptr::null_mut())
+        .unwrap_or_else(ptr::null_mut)
 }
 
 #[no_mangle]
@@ -705,6 +736,25 @@ pub unsafe extern "C" fn dc_send_text_msg(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_send_videochat_invitation(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_send_videochat_invitation()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        chat::send_videochat_invitation(&ctx, ChatId::new(chat_id))
+            .await
+            .map(|msg_id| msg_id.to_u32())
+            .unwrap_or_log_default(&ctx, "Failed to send video chat invitation")
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -830,14 +880,11 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
     };
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_msgs(&ctx, ChatId::new(chat_id), flags, marker_flag)
                 .await
-                .iter()
-                .map(|msg_id| msg_id.to_u32())
-                .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+                .into(),
+        ))
     })
 }
 
@@ -966,7 +1013,7 @@ pub unsafe extern "C" fn dc_get_chat_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
 
     block_on(async move {
-        let arr = dc_array_t::from(
+        Box::into_raw(Box::new(
             chat::get_chat_media(
                 &ctx,
                 ChatId::new(chat_id),
@@ -975,11 +1022,8 @@ pub unsafe extern "C" fn dc_get_chat_media(
                 or_msg_type3,
             )
             .await
-            .iter()
-            .map(|msg_id| msg_id.to_u32())
-            .collect::<Vec<u32>>(),
-        );
-        Box::into_raw(Box::new(arr))
+            .into(),
+        ))
     })
 }
 
@@ -1267,7 +1311,9 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     let muteDuration = match duration {
         0 => MuteDuration::NotMuted,
         -1 => MuteDuration::Forever,
-        n if n > 0 => MuteDuration::Until(SystemTime::now() + Duration::from_secs(duration as u64)),
+        n if n > 0 => SystemTime::now()
+            .checked_add(Duration::from_secs(duration as u64))
+            .map_or(MuteDuration::Forever, MuteDuration::Until),
         _ => {
             warn!(
                 ctx,
@@ -1285,6 +1331,49 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    // Timer value 0 is returned in the rare case of a database error,
+    // but it is not dangerous since it is only meant to be used as a
+    // default when changing the value. Such errors should not be
+    // ignored when ephemeral timer value is used to construct
+    // message headers.
+    block_on(async move { ChatId::new(chat_id).get_ephemeral_timer(ctx).await })
+        .log_err(ctx, "Failed to get ephemeral timer")
+        .unwrap_or_default()
+        .to_u32()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_chat_ephemeral_timer(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    timer: u32,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_set_chat_ephemeral_timer()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .set_ephemeral_timer(ctx, EphemeralTimer::from_u32(timer))
+            .await
+            .log_err(ctx, "Failed to set ephemeral timer")
+            .is_ok() as libc::c_int
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_msg_info(
     context: *mut dc_context_t,
@@ -1980,7 +2069,7 @@ pub unsafe extern "C" fn dc_array_get_timestamp(
         return 0;
     }
 
-    (*array).get_location(index).timestamp
+    (*array).get_timestamp(index).unwrap_or_default()
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_chat_id(
@@ -2027,7 +2116,7 @@ pub unsafe extern "C" fn dc_array_get_marker(
         return std::ptr::null_mut(); // NULL explicitly defined as "no markers"
     }
 
-    if let Some(s) = &(*array).get_location(index).marker {
+    if let Some(s) = (*array).get_marker(index) {
         s.strdup()
     } else {
         std::ptr::null_mut()
@@ -2055,16 +2144,6 @@ pub unsafe extern "C" fn dc_array_search_id(
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_array_get_raw(array: *const dc_array_t) -> *const u32 {
-    if array.is_null() {
-        eprintln!("ignoring careless call to dc_array_get_raw()");
-        return ptr::null_mut();
-    }
-
-    (*array).as_ptr()
-}
-
 // Return the independent-state of the location at the given index.
 // Independent locations do not belong to the track of the user.
 // Returns 1 if location belongs to the track of the user,
@@ -2176,6 +2255,24 @@ pub unsafe extern "C" fn dc_chatlist_get_summary(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chatlist_get_summary2(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    msg_id: u32,
+) -> *mut dc_lot_t {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_chatlist_get_summary2()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+    block_on(async move {
+        let lot =
+            Chatlist::get_summary2(&ctx, ChatId::new(chat_id), MsgId::new(msg_id), None).await;
+        Box::into_raw(Box::new(lot))
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chatlist_get_context(
     chatlist: *mut dc_chatlist_t,
@@ -2373,7 +2470,7 @@ pub unsafe extern "C" fn dc_chat_get_remaining_mute_duration(chat: *mut dc_chat_
         MuteDuration::NotMuted => 0,
         MuteDuration::Forever => -1,
         MuteDuration::Until(when) => when
-            .duration_since(SystemTime::UNIX_EPOCH)
+            .duration_since(SystemTime::now())
             .map(|d| d.as_secs() as i64)
             .unwrap_or(0),
     }
@@ -2644,6 +2741,26 @@ pub unsafe extern "C" fn dc_msg_get_showpadlock(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.get_showpadlock() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_ephemeral_timer(msg: *mut dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_ephemeral_timer()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_ephemeral_timer()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_ephemeral_timestamp(msg: *mut dc_msg_t) -> i64 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_ephemeral_timer()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_ephemeral_timestamp()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_summary(
     msg: *mut dc_msg_t,
@@ -2768,6 +2885,31 @@ pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.is_setupmessage().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_videochat_url()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+
+    ffi_msg
+        .message
+        .get_videochat_url()
+        .unwrap_or_default()
+        .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_videochat_type(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_videochat_type()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_videochat_type().unwrap_or_default() as i32
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_setupcodebegin(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -3218,3 +3360,250 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
     // currently, there is nothing to free, the provider info is a static object.
     // this may change once we start localizing string.
 }
+
+// -- Accounts
+
+/// Struct representing a list of deltachat accounts.
+pub type dc_accounts_t = Accounts;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_new(
+    os_name: *const libc::c_char,
+    dbfile: *const libc::c_char,
+) -> *mut dc_accounts_t {
+    setup_panic!();
+
+    if dbfile.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_new()");
+        return ptr::null_mut();
+    }
+
+    let os_name = if os_name.is_null() {
+        String::from("DcFFI")
+    } else {
+        to_string_lossy(os_name)
+    };
+
+    let accs = block_on(Accounts::new(os_name, as_path(dbfile).to_path_buf().into()));
+
+    match accs {
+        Ok(accs) => Box::into_raw(Box::new(accs)),
+        Err(err) => {
+            eprintln!("failed to create accounts: {}", err);
+            ptr::null_mut()
+        }
+    }
+}
+
+/// Release the accounts structure.
+///
+/// This function releases the memory of the `dc_accounts_t` structure.
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_unref(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_unref()");
+        return;
+    }
+    let _ = Box::from_raw(accounts);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_account(
+    accounts: *mut dc_accounts_t,
+    id: u32,
+) -> *mut dc_context_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_account()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.get_account(id))
+        .map(|ctx| Box::into_raw(Box::new(ctx)))
+        .unwrap_or_else(std::ptr::null_mut)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_selected_account(
+    accounts: *mut dc_accounts_t,
+) -> *mut dc_context_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_selected_account()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    let ctx = block_on(accounts.get_selected_account());
+    Box::into_raw(Box::new(ctx))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_select_account(
+    accounts: *mut dc_accounts_t,
+    id: u32,
+) -> libc::c_int {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_select_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.select_account(id))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -> u32 {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_add_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+
+    block_on(accounts.add_account()).unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_remove_account(
+    accounts: *mut dc_accounts_t,
+    id: u32,
+) -> libc::c_int {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_remove_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+
+    block_on(accounts.remove_account(id))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_migrate_account(
+    accounts: *mut dc_accounts_t,
+    dbfile: *const libc::c_char,
+) -> u32 {
+    if accounts.is_null() || dbfile.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_migrate_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    let dbfile = to_string_lossy(dbfile);
+
+    block_on(accounts.migrate_account(async_std::path::PathBuf::from(dbfile)))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_all(accounts: *mut dc_accounts_t) -> *mut dc_array_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_all()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    let list = block_on(accounts.get_all());
+    let array: dc_array_t = list.into();
+
+    Box::into_raw(Box::new(array))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_import_account(
+    accounts: *mut dc_accounts_t,
+    file: *const libc::c_char,
+) -> u32 {
+    if accounts.is_null() || file.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_import_account()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    let file = to_string_lossy(file);
+    block_on(accounts.import_account(async_std::path::PathBuf::from(file)))
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_start_io()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.start_io());
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_stop_io(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_stop_io()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.stop_io());
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_maybe_network(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_mabye_network()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.maybe_network());
+}
+
+#[no_mangle]
+pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_event_emitter(
+    accounts: *mut dc_accounts_t,
+) -> *mut dc_accounts_event_emitter_t {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_event_emitter()");
+        return ptr::null_mut();
+    }
+
+    let accounts = &*accounts;
+    let emitter = block_on(accounts.get_event_emitter());
+
+    Box::into_raw(Box::new(emitter))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_event_emitter_unref(
+    emitter: *mut dc_accounts_event_emitter_t,
+) {
+    if emitter.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_event_emitter_unref()");
+        return;
+    }
+    let _ = Box::from_raw(emitter);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_get_next_event(
+    emitter: *mut dc_accounts_event_emitter_t,
+) -> *mut dc_event_t {
+    if emitter.is_null() {
+        return ptr::null_mut();
+    }
+    let emitter = &*emitter;
+
+    emitter
+        .recv_sync()
+        .map(|ev| Box::into_raw(Box::new(ev)))
+        .unwrap_or_else(ptr::null_mut)
+}

deltachat-ffi/src/string.rs

@@ -105,8 +105,9 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        CString::new(self.as_ref().as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         })
     }
 
@@ -122,8 +123,9 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         }),
         None => Err(CStringError::NotUnicode),
     }

draft/msgwork_new_imap_jobs.rst

@@ -0,0 +1,66 @@
+
+simplify/streamline mark-seen/delete/move/send-mdn job handling
+---------------------------------------------------------------
+
+Idea: Introduce a new "msgwork" sql table that looks very
+much like the jobs table but has a primary key "msgid"
+and no job id and no foreign-id anymore. This opens up
+bulk-processing by looking at the whole table and combining
+flag-setting to reduce imap-roundtrips and select-folder calls.
+
+Concretely, these IMAP jobs:
+
+    DeleteMsgOnImap
+    MarkseenMsgOnImap
+    MoveMsg
+
+Would be replaced by a few per-message columns in the new msgwork table:
+
+- needs_mark_seen: (bool) message shall be marked as seen on imap
+- needs_to_move: (bool) message should be moved to mvbox_folder
+- deletion_time: (target_time or 0) message shall be deleted at specified time
+- needs_send_mdn: (bool) MDN shall be sent
+
+The various places that currently add the (replaced) jobs
+would now add/modify the respective message record in the message-work table.
+
+Looking at a single message-work entry conceptually looks like this::
+
+    if msg.server_uid==0:
+        return RetryLater  # nothing can be done without server_uid
+
+    if msg.deletion_time > current_time:
+        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
+        clear(needs_deletion)
+        clear(mark_seen)
+
+    if needs_mark_seen:
+        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
+        clear(needs_mark_seen)
+
+    if needs_send_mdn:
+        schedule_smtp_send_mdn(msg)
+        clear(needs_send_mdn)
+
+    if any_flag_set():
+        retrylater
+    # remove msgwork entry from table
+
+
+Notes/Questions:
+
+- it's unclear how much we need per-message retry-time tracking/backoff
+
+- drafting bulk processing algo is useful before
+  going for the implementation, i.e. including select_folder calls etc.
+
+- maybe it's better to not have bools for the flags but
+
+  0 (no change)
+  1 (set the imap flag)
+  2 (clear the imap flag)
+
+  and design such that we can cover all imap flags.
+
+- It might not be neccessary to keep needs_send_mdn state in this table
+  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/cmdline.rs

@@ -1,8 +1,10 @@
+extern crate dirs;
+
 use std::str::FromStr;
 
 use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
+use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -17,7 +19,7 @@ use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::Event;
+use deltachat::EventType;
 use deltachat::{config, provider};
 
 /// Reset database tables.
@@ -86,7 +88,7 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -157,7 +159,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -183,7 +185,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -202,6 +204,15 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.get_viewtype() == Viewtype::VideochatInvitation {
+            format!(
+                "[VIDEOCHAT-INVITATION: {}, type={}]",
+                msg.get_videochat_url().unwrap_or_default(),
+                msg.get_videochat_type().unwrap_or_default()
+            )
+        } else {
+            "".to_string()
+        },
         if msg.is_forwarded() {
             "[FORWARDED]"
         } else {
@@ -215,7 +226,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
 async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id.is_daymarker() {
+        if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
             println!(
                 "--------------------------------------------------------------------------------"
             );
@@ -275,7 +286,9 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, &addr).await;
+            let peerstate = Peerstate::from_addr(context, &addr)
+                .await
+                .expect("peerstate error");
             if peerstate.is_some() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
@@ -359,6 +372,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
                  listmedia\n\
@@ -430,17 +444,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
             imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
-            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -574,6 +592,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
+            let msglist: Vec<MsgId> = msglist
+                .into_iter()
+                .map(|x| match x {
+                    ChatItem::Message { msg_id } => msg_id,
+                    ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
+                    ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
+                })
+                .collect();
+
             let members = chat::get_chat_contacts(&context, sel_chat.id).await;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
@@ -799,6 +826,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "videochat" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
+        }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
@@ -1034,7 +1065,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         // "event" => {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
         //     let event = arg1.parse()?;
-        //     let event = Event::from_u32(event).ok_or(format_err!("Event::from_u32({})", event))?;
+        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
         //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",

examples/repl/main.rs

@@ -19,7 +19,7 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::Event;
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
@@ -34,35 +34,35 @@ mod cmdline;
 use self::cmdline::*;
 
 /// Event Handler
-fn receive_event(event: Event) {
+fn receive_event(event: EventType) {
     let yellow = Color::Yellow.normal();
     match event {
-        Event::Info(msg) => {
+        EventType::Info(msg) => {
             /* do not show the event as this would fill the screen */
             info!("{}", msg);
         }
-        Event::SmtpConnected(msg) => {
+        EventType::SmtpConnected(msg) => {
             info!("[SMTP_CONNECTED] {}", msg);
         }
-        Event::ImapConnected(msg) => {
+        EventType::ImapConnected(msg) => {
             info!("[IMAP_CONNECTED] {}", msg);
         }
-        Event::SmtpMessageSent(msg) => {
+        EventType::SmtpMessageSent(msg) => {
             info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
-        Event::Warning(msg) => {
+        EventType::Warning(msg) => {
             warn!("{}", msg);
         }
-        Event::Error(msg) => {
+        EventType::Error(msg) => {
             error!("{}", msg);
         }
-        Event::ErrorNetwork(msg) => {
+        EventType::ErrorNetwork(msg) => {
             error!("[NETWORK] msg={}", msg);
         }
-        Event::ErrorSelfNotInGroup(msg) => {
+        EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
-        Event::MsgsChanged { chat_id, msg_id } => {
+        EventType::MsgsChanged { chat_id, msg_id } => {
             info!(
                 "{}",
                 yellow.paint(format!(
@@ -71,34 +71,34 @@ fn receive_event(event: Event) {
                 ))
             );
         }
-        Event::ContactsChanged(_) => {
+        EventType::ContactsChanged(_) => {
             info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
-        Event::LocationChanged(contact) => {
+        EventType::LocationChanged(contact) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        Event::ConfigureProgress(progress) => {
+        EventType::ConfigureProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
             );
         }
-        Event::ImexProgress(progress) => {
+        EventType::ImexProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        Event::ImexFileWritten(file) => {
+        EventType::ImexFileWritten(file) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
-        Event::ChatModified(chat) => {
+        EventType::ChatModified(chat) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
@@ -158,7 +158,7 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 27] = [
     "listchats",
     "listarchived",
     "chat",
@@ -178,6 +178,7 @@ const CHAT_COMMANDS: [&str; 26] = [
     "send",
     "sendimage",
     "sendfile",
+    "videochat",
     "draft",
     "listmedia",
     "archive",
@@ -271,12 +272,12 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            receive_event(event);
+            receive_event(event.typ);
         }
     });
 

examples/simple.rs

@@ -6,20 +6,20 @@ use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::Event;
+use deltachat::EventType;
 
-fn cb(event: Event) {
+fn cb(event: EventType) {
     match event {
-        Event::ConfigureProgress(progress) => {
+        EventType::ConfigureProgress(progress) => {
             log::info!("progress: {}", progress);
         }
-        Event::Info(msg) => {
+        EventType::Info(msg) => {
             log::info!("{}", msg);
         }
-        Event::Warning(msg) => {
+        EventType::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        Event::Error(msg) | Event::ErrorNetwork(msg) => {
+        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into())
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
@@ -45,7 +45,7 @@ async fn main() {
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            cb(event);
+            cb(event.typ);
         }
     });
 

python/CHANGELOG

@@ -1,6 +1,22 @@
-0.900.0 (DRAFT)
+1.44.0
+------
+
+- fix Chat.get_mute_duration()
+
+1.40.1
 ---------------
 
+- emit "ac_member_removed" event (with 'actor' being the removed contact) 
+  for when a user leaves a group.
+
+- fix create_contact(addr) when addr is the self-contact. 
+
+
+1.40.0
+---------------
+
+- uses latest 1.40+ Delta Chat core
+
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -10,6 +26,7 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
+
 0.800.0
 -------
 

python/README.rst

@@ -7,76 +7,14 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
-Installing bindings from source (Updated: 20-Jan-2020)
-=========================================================
-
-Install Rust and Cargo first.  Deltachat needs a specific nightly
-version, the easiest is probably to first install Rust stable from
-rustup and then use this to install the correct nightly version.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then GIT clone the deltachat-core-rust repo and get the actual
-rust- and cargo-toolchain needed by deltachat::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-   rustup show
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   ./install_python_bindings.py
-
-The installation might take a while, depending on your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_NEW_TMP_EMAIL`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs. This is implemented on the server side via
-  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
+without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
+yourself <sourceinstall>`_.
 
-We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -103,6 +41,78 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
+Running tests
+=============
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+This will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real e-mail servers.
+
+.. _livetests:
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
+
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
+
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
+One hour is enough to invoke pytest and run all offline and online tests:
+
+    pytest
+
+    # or if you have installed pytest-xdist for parallel test execution
+    pytest -n6
+
+Each test run creates new accounts.
+
+
+.. _sourceinstall:
+
+Installing bindings from source (Updated: July 2020)
+=========================================================
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   python install_python_bindings.py
+
+The core compilation and bindings building might take a while,
+depending on the speed of your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+
 Code examples
 =============
 

python/doc/_templates/globaltoc.html

@@ -9,8 +9,7 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
-  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
+  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/group_tracking.py

@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
+    def ac_member_added(self, chat, contact, actor, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
+    def ac_member_removed(self, chat, contact, actor, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
+        *ac_member_added {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))
+        *ac_member_removed {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))

python/install_python_bindings.py

@@ -17,8 +17,12 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+
     if target == 'release':
+        extra = " -C lto=on -C embed-bitcode=yes"
+        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
         cmd.append("--release")
+
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)

python/src/deltachat/account.py

@@ -246,7 +246,6 @@ class Account(object):
         addr = as_dc_charpointer(addr)
         name = as_dc_charpointer(name)
         contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -607,12 +606,24 @@ class Account(object):
         self.stop_io()
 
         self.log("remove dc_context references")
-        # the dc_context_unref triggers get_next_event to return ffi.NULL
-        # which in turns makes the event thread finish execution
+
+        # if _dc_context is unref'ed the event thread should quickly
+        # receive the termination signal. However, some python code might
+        # still hold a reference and so we use a secondary signal
+        # to make sure the even thread terminates if it receives any new
+        # event, indepedently from waiting for the core to send NULL to
+        # get_next_event().
+        self._event_thread.mark_shutdown()
         self._dc_context = None
 
         self.log("wait for event thread to finish")
-        self._event_thread.wait()
+        try:
+            self._event_thread.wait(timeout=2)
+        except RuntimeError as e:
+            self.log("Waiting for event thread failed: {}".format(e))
+
+        if self._event_thread.is_alive():
+            self.log("WARN: event thread did not terminate yet, ignoring.")
 
         self._shutdown_event.set()
 

python/src/deltachat/chat.py

@@ -137,7 +137,23 @@ class Chat(object):
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
+        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
+
+    def get_ephemeral_timer(self):
+        """ get ephemeral timer.
+
+        :returns: ephemeral timer value in seconds
+        """
+        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
+
+    def set_ephemeral_timer(self, timer):
+        """ set ephemeral timer.
+
+        :param: timer value in seconds
+
+        :returns: None
+        """
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
 
     def get_type(self):
         """ (deprecated) return type of this chat.

python/src/deltachat/direct_imap.py

@@ -108,6 +108,15 @@ class DirectImap:
 
     def get_all_messages(self):
         assert not self._idling
+
+        # Flush unsolicited responses. IMAPClient has problems
+        # dealing with them: https://github.com/mjs/imapclient/issues/334
+        # When this NOOP was introduced, next FETCH returned empty
+        # result instead of a single message, even though IMAP server
+        # can only return more untagged responses than required, not
+        # less.
+        self.conn.noop()
+
         return self.conn.fetch(ALL, [FLAGS])
 
     def get_unread_messages(self):
@@ -159,9 +168,13 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            requested = [b'BODY.PEEK[]', FLAGS]
             for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[HEADER]']
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
+                    continue
+
                 flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
@@ -192,6 +205,7 @@ class DirectImap:
             raise TimeoutError
         if terminate:
             self.idle_done()
+        self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
     def idle_wait_for_seen(self):

python/src/deltachat/events.py

@@ -86,11 +86,11 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
+    def get_info_contains(self, regex):
+        rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
+            if rex.search(ev.data2):
                 return ev
 
     def ensure_event_not_queued(self, event_name_regex):
@@ -139,6 +139,7 @@ class EventThread(threading.Thread):
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
+        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -147,12 +148,15 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def wait(self):
+    def mark_shutdown(self):
+        self._marked_for_shutdown = True
+
+    def wait(self, timeout=None):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join()
+        self.join(timeout=timeout)
 
     def run(self):
         """ get and run events until shutdown. """
@@ -164,10 +168,12 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while 1:
+        while not self._marked_for_shutdown:
             event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
+            if self._marked_for_shutdown:
+                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are not allowed to block/last long as this
+    Hooks are generally not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,10 +31,6 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -55,19 +51,37 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
+        """ Called when an outgoing message has been delivered to SMTP.
+
+        :param message: Message that was just delivered.
+        """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
+        """ Chat was created or modified regarding membership, avatar, title.
+
+        :param chat: Chat which was modified.
+        """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
+    def ac_member_added(self, chat, contact, actor, message):
+        """ Called for each contact added to an accepted chat.
+
+        :param chat: Chat where contact was added.
+        :param contact: Contact that was added.
+        :param actor: Who added the contact (None if it was our self-addr)
+        :param message: The original system message that reports the addition.
+        """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
+    def ac_member_removed(self, chat, contact, actor, message):
+        """ Called for each contact removed from a chat.
+
+        :param chat: Chat where contact was removed.
+        :param contact: Contact that was removed.
+        :param actor: Who removed the contact (None if it was our self-addr)
+        :param message: The original system message that reports the removal.
+        """
 
 
 class Global:

python/src/deltachat/message.py

@@ -1,6 +1,7 @@
 """ The Message object. """
 
 import os
+import re
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -154,6 +155,26 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @props.with_doc
+    def ephemeral_timer(self):
+        """Ephemeral timer in seconds
+
+        :returns: timer in seconds or None if there is no timer
+        """
+        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
+        if timer:
+            return timer
+
+    @props.with_doc
+    def ephemeral_timestamp(self):
+        """UTC time when the message will be deleted.
+
+        :returns: naive datetime.datetime() object or None if the timer is not started.
+        """
+        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
+        if ts:
+            return datetime.utcfromtimestamp(ts)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -336,20 +357,43 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
+        if not res:
+            return
+        action, affected, actor = res
+        affected = msg.account.get_contact_by_addr(affected)
+        if actor == "me":
+            actor = None
+        else:
+            actor = msg.account.get_contact_by_addr(actor)
+        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
+        return "ac_member_" + res[0], d
+
+
+def extract_addr(text):
+    m = re.match(r'.*\((.+@.+)\)', text)
+    if m:
+        text = m.group(1)
+    text = text.rstrip(".")
+    return text.strip()
 
 
 def parse_system_add_remove(text):
+    """ return add/remove info from parsing the given system message text.
+
+    returns a (action, affected, actor) triple """
+
     # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
+    # Member x@y added by a@b
+    # Member With space (tmp1@x.org) removed by tmp2@x.org.
+    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+    # Group left by some one (tmp1@x.org).
+    # Group left by tmp1@x.org.
     text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")
+    m = re.match(r'member (.+) (removed|added) by (.+)', text)
+    if m:
+        affected, action, actor = m.groups()
+        return action, extract_addr(affected), extract_addr(actor)
+    if text.startswith("group left by "):
+        addr = extract_addr(text[13:])
+        if addr:
+            return "removed", addr, addr

python/src/deltachat/testplugin.py

@@ -32,6 +32,10 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
+    parser.addoption(
+        "--strict-tls", action="store_true",
+        help="Never accept invalid TLS certificates for test accounts",
+    )
 
 
 def pytest_configure(config):
@@ -152,7 +156,7 @@ class SessionLiveConfigFromURL:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
+                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
@@ -231,10 +235,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
             ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac))
+                logger = FFIEventLogger(ac)
+                logger.init_time = self.init_time
+                ac.add_account_plugin(logger)
             self._accounts.append(ac)
             return ac
 
@@ -244,10 +251,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
+            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -282,16 +286,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            if pytestconfig.getoption("--strict-tls"):
+                # Enable strict certificate checks for online accounts
+                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
@@ -332,8 +335,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),

python/tests/test_account.py

@@ -11,8 +11,21 @@ from datetime import datetime, timedelta
 
 
 @pytest.mark.parametrize("msgtext,res", [
-    ("Member Me (tmp1@x.org) removed by tmp2@x.org.", ("removed", "tmp1@x.org")),
-    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org")),
+    ("Member Me (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by me",
+        ("removed", "tmp1@x.org", "me")),
+    ("Group left by some one (tmp1@x.org).",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Group left by tmp1@x.org.",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org", "tmp2@x.org")),
+    ("Member nothing bla bla", None),
+    ("Another unknown system message", None),
 ])
 def test_parse_system_add_remove(msgtext, res):
     from deltachat.message import parse_system_add_remove
@@ -116,6 +129,11 @@ class TestOfflineContact:
         assert not contact1.is_blocked()
         assert not contact1.is_verified()
 
+    def test_create_self_contact(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact(ac1.get_config("addr"))
+        assert contact1.id == 1
+
     def test_get_contacts_and_delete(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact("some1@example.org", name="some1")
@@ -258,15 +276,23 @@ class TestOfflineChat:
     def test_mute(self, ac1):
         chat = ac1.create_group_chat(name="title1")
         assert not chat.is_muted()
+        assert chat.get_mute_duration() == 0
         chat.mute()
         assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
         chat.unmute()
         assert not chat.is_muted()
         chat.mute(50)
         assert chat.is_muted()
+        assert chat.get_mute_duration() <= 50
         with pytest.raises(ValueError):
             chat.mute(-51)
 
+        # Regression test, this caused Rust panic previously
+        chat.mute(2**63 - 1)
+        assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
+
     def test_delete_and_send_fails(self, ac1, chat1):
         chat1.delete()
         ac1._evtracker.wait_next_messages_changed()
@@ -452,12 +478,12 @@ class TestOfflineChat:
 
         class InPlugin:
             @account_hookimpl
-            def ac_member_added(self, chat, contact):
-                in_list.append(("added", chat, contact))
+            def ac_member_added(self, chat, contact, actor):
+                in_list.append(("added", chat, contact, actor))
 
             @account_hookimpl
-            def ac_member_removed(self, chat, contact):
-                in_list.append(("removed", chat, contact))
+            def ac_member_removed(self, chat, contact, actor):
+                in_list.append(("removed", chat, contact, actor))
 
         ac1.add_account_plugin(InPlugin())
 
@@ -486,10 +512,11 @@ class TestOfflineChat:
 
         assert len(in_list) == 10
         chat_contacts = chat.get_contacts()
-        for in_cmd, in_chat, in_contact in in_list:
+        for in_cmd, in_chat, in_contact, in_actor in in_list:
             assert in_cmd == "added"
             assert in_chat == chat
             assert in_contact in chat_contacts
+            assert in_actor is None
             chat_contacts.remove(in_contact)
 
         assert chat_contacts[0].id == 1  # self contact
@@ -517,7 +544,7 @@ def test_basic_imap_api(acfactory, tmpdir):
 
     imap2 = ac2.direct_imap
 
-    ac2.direct_imap.idle_start()
+    imap2.idle_start()
     chat12.send_text("hello")
     ac2._evtracker.wait_next_incoming_message()
 
@@ -1542,6 +1569,127 @@ class TestOnlineAccount:
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac2.create_chat(ac4)
 
+    def test_immediate_autodelete(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account(mvbox=False, move=False, sentbox=False)
+
+        # "1" means delete immediately, while "0" means do not delete
+        ac2.set_config("delete_server_after", "1")
+
+        acfactory.wait_configure_and_start_io()
+
+        imap2 = ac2.direct_imap
+        imap2.idle_start()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+        ac2.create_chat(ac1)
+
+        sent_msg = chat1.send_text("hello")
+        imap2.idle_check(terminate=False)
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        imap2.idle_check(terminate=True)
+        ac2._evtracker.get_info_contains("close/expunge succeeded")
+
+        assert len(imap2.get_all_messages()) == 0
+
+        # Mark deleted message as seen and check that read receipt arrives
+        msg.mark_seen()
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
+        assert ev.data1 == chat1.id
+        assert ev.data2 == sent_msg.id
+
+    def test_ephemeral_timer(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+        chat2 = ac2.create_chat(ac1)
+
+        lp.sec("ac1: set ephemeral timer to 60")
+        chat1.set_ephemeral_timer(60)
+
+        lp.sec("ac1: check that ephemeral timer is set for chat")
+        assert chat1.get_ephemeral_timer() == 60
+        chat1_summary = chat1.get_summary()
+        assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
+
+        lp.sec("ac2: receive system message about ephemeral timer modification")
+        ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        system_message1 = ac2._evtracker.wait_next_incoming_message()
+        assert chat2.get_ephemeral_timer() == 60
+        assert system_message1.is_system_message()
+
+        # Disabled until markers are implemented
+        # assert "Ephemeral timer: 60\n" in system_message1.get_message_info()
+
+        lp.sec("ac2: send message to ac1")
+        sent_message = chat2.send_text("message")
+        assert sent_message.ephemeral_timer == 60
+        assert "Ephemeral timer: 60\n" in sent_message.get_message_info()
+
+        # Timer is started immediately for sent messages
+        assert sent_message.ephemeral_timestamp is not None
+        assert "Expires: " in sent_message.get_message_info()
+
+        lp.sec("ac1: waiting for message from ac2")
+        text_message = ac1._evtracker.wait_next_incoming_message()
+        assert text_message.text == "message"
+        assert text_message.ephemeral_timer == 60
+        assert "Ephemeral timer: 60\n" in text_message.get_message_info()
+
+        # Timer should not start until message is displayed
+        assert text_message.ephemeral_timestamp is None
+        assert "Expires: " not in text_message.get_message_info()
+        text_message.mark_seen()
+        text_message = ac1.get_message_by_id(text_message.id)
+        assert text_message.ephemeral_timestamp is not None
+        assert "Expires: " in text_message.get_message_info()
+
+        lp.sec("ac2: set ephemeral timer to 0")
+        chat2.set_ephemeral_timer(0)
+        ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+
+        lp.sec("ac1: receive system message about ephemeral timer modification")
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        system_message2 = ac1._evtracker.wait_next_incoming_message()
+        assert system_message2.ephemeral_timer is None
+        assert "Ephemeral timer: " not in system_message2.get_message_info()
+        assert chat1.get_ephemeral_timer() == 0
+
+    def test_delete_multiple_messages(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("ac1: sending seven messages")
+        texts = ["first", "second", "third", "fourth", "fifth", "sixth", "seventh"]
+        for text in texts:
+            chat12.send_text(text)
+
+        lp.sec("ac2: waiting for all messages on the other side")
+        to_delete = []
+        for text in texts:
+            msg = ac2._evtracker.wait_next_incoming_message()
+            assert msg.text in texts
+            if text != "third":
+                to_delete.append(msg)
+
+        lp.sec("ac2: deleting all messages except third")
+        assert len(to_delete) == len(texts) - 1
+        ac2.delete_messages(to_delete)
+        for msg in to_delete:
+            ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+
+        ac2._evtracker.get_info_contains("close/expunge succeeded")
+
+        lp.sec("imap2: test that only one message is left")
+        imap2 = ac2.direct_imap
+
+        assert len(imap2.get_all_messages()) == 1
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):

python/tox.ini

@@ -7,7 +7,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

rust-toolchain

@@ -1 +1 @@
-1.43.1
+1.45.0

set_core_version.py

@@ -5,18 +5,27 @@ import sys
 import re
 import pathlib
 import subprocess
+from argparse import ArgumentParser
 
 rex = re.compile(r'version = "(\S+)"')
 
-def read_toml_version(relpath):
+
+def regex_matches(relpath, regex=rex):
     p = pathlib.Path(relpath)
     assert p.exists()
     for line in open(str(p)):
-        m = rex.match(line)
+        m = regex.match(line)
         if m is not None:
-            return m.group(1)
+            return m
+
+
+def read_toml_version(relpath):
+    res = regex_matches(relpath, rex)
+    if res is not None:
+        return res.group(1)
     raise ValueError("no version found in {}".format(relpath))
 
+
 def replace_toml_version(relpath, newversion):
     p = pathlib.Path(relpath)
     assert p.exists()
@@ -25,18 +34,28 @@ def replace_toml_version(relpath, newversion):
         for line in open(str(p)):
             m = rex.match(line)
             if m is not None:
+                print("{}: set version={}".format(relpath, newversion))
                 f.write('version = "{}"\n'.format(newversion))
             else:
                 f.write(line)
     os.rename(tmp_path, str(p))
 
-if __name__ == "__main__":
 
-    if len(sys.argv) < 2:
-        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
+def main():
+    parser = ArgumentParser(prog="set_core_version")
+    parser.add_argument("newversion")
+
+    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml"]
+    try:
+        opts = parser.parse_args()
+    except SystemExit:
+        print()
+        for x in toml_list:
             print("{}: {}".format(x, read_toml_version(x)))
+        print()
         raise SystemExit("need argument: new version, example: 1.25.0")
-    newversion = sys.argv[1]
+
+    newversion = opts.newversion
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
 
@@ -55,7 +74,10 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
+    print("running cargo check")
     subprocess.call(["cargo", "check"])
+
+    print("adding changes to git index")
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
@@ -63,3 +85,8 @@ if __name__ == "__main__":
     print("")
     print("   git tag {}".format(newversion))
     print("")
+
+
+if __name__ == "__main__":
+    main()
+

src/accounts.rs

@@ -0,0 +1,533 @@
+use std::collections::HashMap;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::task::{Context as TaskContext, Poll};
+
+use async_std::fs;
+use async_std::path::PathBuf;
+use async_std::sync::{Arc, RwLock};
+use uuid::Uuid;
+
+use anyhow::{ensure, Context as _};
+use serde::{Deserialize, Serialize};
+
+use crate::context::Context;
+use crate::error::Result;
+use crate::events::Event;
+
+/// Account manager, that can handle multiple accounts in a single place.
+#[derive(Debug, Clone)]
+pub struct Accounts {
+    dir: PathBuf,
+    config: Config,
+    accounts: Arc<RwLock<HashMap<u32, Context>>>,
+}
+
+impl Accounts {
+    /// Loads or creates an accounts folder at the given `dir`.
+    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
+        if !dir.exists().await {
+            Accounts::create(os_name, &dir).await?;
+        }
+
+        Accounts::open(dir).await
+    }
+
+    /// Creates a new default structure, including a default account.
+    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
+        fs::create_dir_all(dir)
+            .await
+            .context("failed to create folder")?;
+
+        // create default account
+        let config = Config::new(os_name.clone(), dir).await?;
+        let account_config = config.new_account(dir).await?;
+
+        Context::new(os_name, account_config.dbfile().into(), account_config.id)
+            .await
+            .context("failed to create default account")?;
+
+        Ok(())
+    }
+
+    /// Opens an existing accounts structure. Will error if the folder doesn't exist,
+    /// no account exists and no config exists.
+    pub async fn open(dir: PathBuf) -> Result<Self> {
+        ensure!(dir.exists().await, "directory does not exist");
+
+        let config_file = dir.join(CONFIG_NAME);
+        ensure!(config_file.exists().await, "accounts.toml does not exist");
+
+        let config = Config::from_file(config_file).await?;
+        let accounts = config.load_accounts().await?;
+
+        Ok(Self {
+            dir,
+            config,
+            accounts: Arc::new(RwLock::new(accounts)),
+        })
+    }
+
+    /// Get an account by its `id`:
+    pub async fn get_account(&self, id: u32) -> Option<Context> {
+        self.accounts.read().await.get(&id).cloned()
+    }
+
+    /// Get the currently selected account.
+    pub async fn get_selected_account(&self) -> Context {
+        let id = self.config.get_selected_account().await;
+        self.accounts
+            .read()
+            .await
+            .get(&id)
+            .cloned()
+            .expect("inconsistent state")
+    }
+
+    /// Select the given account.
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        self.config.select_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Add a new account.
+    pub async fn add_account(&self) -> Result<u32> {
+        let os_name = self.config.os_name().await;
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        self.accounts.write().await.insert(account_config.id, ctx);
+
+        Ok(account_config.id)
+    }
+
+    /// Remove an account.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        let ctx = self.accounts.write().await.remove(&id);
+        ensure!(ctx.is_some(), "no account with this id: {}", id);
+        let ctx = ctx.unwrap();
+        ctx.stop_io().await;
+        drop(ctx);
+
+        if let Some(cfg) = self.config.get_account(id).await {
+            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                .await
+                .context("failed to remove account data")?;
+        }
+        self.config.remove_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Migrate an existing account into this structure.
+    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
+        let blobdir = Context::derive_blobdir(&dbfile);
+
+        ensure!(
+            dbfile.exists().await,
+            "no database found: {}",
+            dbfile.display()
+        );
+        ensure!(
+            blobdir.exists().await,
+            "no blobdir found: {}",
+            blobdir.display()
+        );
+
+        let old_id = self.config.get_selected_account().await;
+
+        // create new account
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let new_dbfile = account_config.dbfile().into();
+        let new_blobdir = Context::derive_blobdir(&new_dbfile);
+
+        let res = {
+            fs::create_dir_all(&account_config.dir).await?;
+            fs::rename(&dbfile, &new_dbfile).await?;
+            fs::rename(&blobdir, &new_blobdir).await?;
+            Ok(())
+        };
+
+        match res {
+            Ok(_) => {
+                let ctx = Context::with_blobdir(
+                    self.config.os_name().await,
+                    new_dbfile,
+                    new_blobdir,
+                    account_config.id,
+                )
+                .await?;
+                self.accounts.write().await.insert(account_config.id, ctx);
+                Ok(account_config.id)
+            }
+            Err(err) => {
+                // remove temp account
+                fs::remove_dir_all(async_std::path::PathBuf::from(&account_config.dir))
+                    .await
+                    .context("failed to remove account data")?;
+
+                self.config.remove_account(account_config.id).await?;
+
+                // set selection back
+                self.select_account(old_id).await?;
+
+                Err(err)
+            }
+        }
+    }
+
+    /// Get a list of all account ids.
+    pub async fn get_all(&self) -> Vec<u32> {
+        self.accounts.read().await.keys().copied().collect()
+    }
+
+    /// Import a backup using a new account and selects it.
+    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
+        let old_id = self.config.get_selected_account().await;
+
+        let id = self.add_account().await?;
+        let ctx = self.get_account(id).await.expect("just added");
+
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, Some(file)).await {
+            Ok(_) => Ok(id),
+            Err(err) => {
+                // remove temp account
+                self.remove_account(id).await?;
+                // set selection back
+                self.select_account(old_id).await?;
+                Err(err)
+            }
+        }
+    }
+
+    pub async fn start_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.start_io().await;
+        }
+    }
+
+    pub async fn stop_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.stop_io().await;
+        }
+    }
+
+    pub async fn maybe_network(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.maybe_network().await;
+        }
+    }
+
+    /// Unified event emitter.
+    pub async fn get_event_emitter(&self) -> EventEmitter {
+        let emitters = self
+            .accounts
+            .read()
+            .await
+            .iter()
+            .map(|(id, a)| EmitterWrapper {
+                id: *id,
+                emitter: a.get_event_emitter(),
+                done: AtomicBool::new(false),
+            })
+            .collect();
+
+        EventEmitter(emitters)
+    }
+}
+
+impl EventEmitter {
+    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
+    pub fn recv_sync(&self) -> Option<Event> {
+        async_std::task::block_on(self.recv())
+    }
+
+    /// Async recv of an event. Return `None` if the `Sender` has been droped.
+    pub async fn recv(&self) -> Option<Event> {
+        futures::future::poll_fn(|cx| Pin::new(self).recv_poll(cx)).await
+    }
+
+    fn recv_poll(self: Pin<&Self>, _cx: &mut TaskContext<'_>) -> Poll<Option<Event>> {
+        for e in &*self.0 {
+            if e.done.load(Ordering::Acquire) {
+                // skip emitters that are already done
+                continue;
+            }
+
+            match e.emitter.try_recv() {
+                Ok(event) => return Poll::Ready(Some(event)),
+                Err(async_std::sync::TryRecvError::Disconnected) => {
+                    e.done.store(true, Ordering::Release);
+                }
+                Err(async_std::sync::TryRecvError::Empty) => {}
+            }
+        }
+
+        Poll::Pending
+    }
+}
+
+#[derive(Debug)]
+pub struct EventEmitter(Vec<EmitterWrapper>);
+
+#[derive(Debug)]
+struct EmitterWrapper {
+    id: u32,
+    emitter: crate::events::EventEmitter,
+    done: AtomicBool,
+}
+
+pub const CONFIG_NAME: &str = "accounts.toml";
+pub const DB_NAME: &str = "dc.db";
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    file: PathBuf,
+    inner: Arc<RwLock<InnerConfig>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+struct InnerConfig {
+    pub os_name: String,
+    /// The currently selected account.
+    pub selected_account: u32,
+    pub next_id: u32,
+    pub accounts: Vec<AccountConfig>,
+}
+
+impl Config {
+    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
+        let cfg = Config {
+            file: dir.join(CONFIG_NAME),
+            inner: Arc::new(RwLock::new(InnerConfig {
+                os_name,
+                accounts: Vec::new(),
+                selected_account: 0,
+                next_id: 1,
+            })),
+        };
+
+        cfg.sync().await?;
+
+        Ok(cfg)
+    }
+
+    pub async fn os_name(&self) -> String {
+        self.inner.read().await.os_name.clone()
+    }
+
+    /// Sync the inmemory representation to disk.
+    async fn sync(&self) -> Result<()> {
+        fs::write(
+            &self.file,
+            toml::to_string_pretty(&*self.inner.read().await)?,
+        )
+        .await
+        .context("failed to write config")
+    }
+
+    /// Read a configuration from the given file into memory.
+    pub async fn from_file(file: PathBuf) -> Result<Self> {
+        let bytes = fs::read(&file).await.context("failed to read file")?;
+        let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
+
+        Ok(Config {
+            file,
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+
+    pub async fn load_accounts(&self) -> Result<HashMap<u32, Context>> {
+        let cfg = &*self.inner.read().await;
+        let mut accounts = HashMap::with_capacity(cfg.accounts.len());
+        for account_config in &cfg.accounts {
+            let ctx = Context::new(
+                cfg.os_name.clone(),
+                account_config.dbfile().into(),
+                account_config.id,
+            )
+            .await?;
+            accounts.insert(account_config.id, ctx);
+        }
+
+        Ok(accounts)
+    }
+
+    /// Create a new account in the given root directory.
+    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+        let id = {
+            let inner = &mut self.inner.write().await;
+            let id = inner.next_id;
+            let uuid = Uuid::new_v4();
+            let target_dir = dir.join(uuid.to_simple_ref().to_string());
+
+            inner.accounts.push(AccountConfig {
+                id,
+                name: String::new(),
+                dir: target_dir.into(),
+                uuid,
+            });
+            inner.next_id += 1;
+            id
+        };
+
+        self.sync().await?;
+
+        self.select_account(id).await.expect("just added");
+        let cfg = self.get_account(id).await.expect("just added");
+        Ok(cfg)
+    }
+
+    /// Removes an existing acccount entirely.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
+                // remove account from the configs
+                inner.accounts.remove(idx);
+            }
+            if inner.selected_account == id {
+                // reset selected account
+                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+            }
+        }
+
+        self.sync().await
+    }
+
+    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+        self.inner
+            .read()
+            .await
+            .accounts
+            .iter()
+            .find(|e| e.id == id)
+            .cloned()
+    }
+
+    pub async fn get_selected_account(&self) -> u32 {
+        self.inner.read().await.selected_account
+    }
+
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            ensure!(
+                inner.accounts.iter().any(|e| e.id == id),
+                "invalid account id: {}",
+                id
+            );
+
+            inner.selected_account = id;
+        }
+
+        self.sync().await?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct AccountConfig {
+    /// Unique id.
+    pub id: u32,
+    /// Display name
+    pub name: String,
+    /// Root directory for all data for this account.
+    pub dir: std::path::PathBuf,
+    pub uuid: Uuid,
+}
+
+impl AccountConfig {
+    /// Get the canoncial dbfile name for this configuration.
+    pub fn dbfile(&self) -> std::path::PathBuf {
+        self.dir.join(DB_NAME)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_account_new_open() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts1").into();
+
+        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let accounts2 = Accounts::open(p).await.unwrap();
+
+        assert_eq!(accounts1.accounts.read().await.len(), 1);
+        assert_eq!(accounts1.config.get_selected_account().await, 1);
+
+        assert_eq!(accounts1.dir, accounts2.dir);
+        assert_eq!(
+            &*accounts1.config.inner.read().await,
+            &*accounts2.config.inner.read().await,
+        );
+        assert_eq!(
+            accounts1.accounts.read().await.len(),
+            accounts2.accounts.read().await.len()
+        );
+    }
+
+    #[async_std::test]
+    async fn test_account_new_add_remove() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, 2);
+        assert_eq!(accounts.config.get_selected_account().await, id);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+
+        accounts.select_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        accounts.remove_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+    }
+
+    #[async_std::test]
+    async fn test_migrate_account() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let extern_dbfile: PathBuf = dir.path().join("other").into();
+        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
+            .await
+            .unwrap();
+        ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
+            .await
+            .unwrap();
+
+        drop(ctx);
+
+        accounts
+            .migrate_account(extern_dbfile.clone())
+            .await
+            .unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+
+        let ctx = accounts.get_selected_account().await;
+        assert_eq!(
+            "me@mail.com",
+            ctx.get_config(crate::config::Config::Addr).await.unwrap()
+        );
+    }
+}

src/blob.rs

@@ -15,7 +15,7 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::error::Error;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::message;
 
 /// Represents a file in the blob directory.
@@ -67,7 +67,7 @@ impl<'a> BlobObject<'a> {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -155,7 +155,7 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -168,6 +168,9 @@ impl<'a> BlobObject<'a> {
     /// subdirectory is used and [BlobObject::sanitise_name] does not
     /// modify the filename.
     ///
+    /// Paths into the blob directory may be either defined by an absolute path
+    /// or by the relative prefix `$BLOBDIR`.
+    ///
     /// # Errors
     ///
     /// This merely delegates to the [BlobObject::create_and_copy] and
@@ -179,6 +182,11 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'_>, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
+        } else if src.as_ref().starts_with("$BLOBDIR/") {
+            BlobObject::from_name(
+                context,
+                src.as_ref().to_str().unwrap_or_default().to_string(),
+            )
         } else {
             BlobObject::create_and_copy(context, src).await
         }

src/chat.rs

@@ -15,8 +15,9 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::{delete_expired_messages, schedule_ephemeral_task, Timer as EphemeralTimer};
 use crate::error::{bail, ensure, format_err, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Action};
 use crate::message::{self, InvalidMsgId, Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -24,6 +25,25 @@ use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
+/// An chat item, such as a message or a marker.
+#[derive(Debug, Copy, Clone)]
+pub enum ChatItem {
+    Message {
+        msg_id: MsgId,
+    },
+
+    /// A marker without inherent meaning. It is inserted before user
+    /// supplied MsgId.
+    Marker1,
+
+    /// Day marker, separating messages that correspond to different
+    /// days according to local time.
+    DayMarker {
+        /// Marker timestamp, for day markers
+        timestamp: i64,
+    },
+}
+
 /// Chat ID, including reserved IDs.
 ///
 /// Some chat IDs are reserved to identify special chat types.  This
@@ -165,7 +185,7 @@ impl ChatId {
             )
             .await?;
 
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             msg_id: MsgId::new(0),
             chat_id: ChatId::new(0),
         });
@@ -222,7 +242,7 @@ impl ChatId {
             .execute("DELETE FROM chats WHERE id=?;", paramsv![self])
             .await?;
 
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             msg_id: MsgId::new(0),
             chat_id: ChatId::new(0),
         });
@@ -248,7 +268,7 @@ impl ChatId {
         };
 
         if changed {
-            context.emit_event(Event::MsgsChanged {
+            context.emit_event(EventType::MsgsChanged {
                 chat_id: self,
                 msg_id: MsgId::new(0),
             });
@@ -710,6 +730,7 @@ impl Chat {
                 .unwrap_or_else(std::path::PathBuf::new),
             draft,
             is_muted: self.is_muted(),
+            ephemeral_timer: self.id.get_ephemeral_timer(context).await?,
         })
     }
 
@@ -768,7 +789,7 @@ impl Chat {
         {
             emit_event!(
                 context,
-                Event::ErrorSelfNotInGroup("Cannot send message; self not in group.".into())
+                EventType::ErrorSelfNotInGroup("Cannot send message; self not in group.".into())
             );
             bail!("Cannot set message; self not in group.");
         }
@@ -886,11 +907,10 @@ impl Chat {
 
                     // the whole list of messages referenced may be huge;
                     // only use the oldest and and the parent message
-                    let parent_references = if let Some(n) = parent_references.find(' ') {
-                        &parent_references[0..n]
-                    } else {
-                        &parent_references
-                    };
+                    let parent_references = parent_references
+                        .find(' ')
+                        .and_then(|n| parent_references.get(..n))
+                        .unwrap_or(&parent_references);
 
                     if !parent_references.is_empty() && !parent_rfc724_mid.is_empty() {
                         // angle brackets are added by the mimefactory later
@@ -938,10 +958,20 @@ impl Chat {
                     .await?;
             }
 
+            let ephemeral_timer = if msg.param.get_cmd() == SystemMessage::EphemeralTimerChanged {
+                EphemeralTimer::Disabled
+            } else {
+                self.id.get_ephemeral_timer(context).await?
+            };
+            let ephemeral_timestamp = match ephemeral_timer {
+                EphemeralTimer::Disabled => 0,
+                EphemeralTimer::Enabled { duration } => timestamp + i64::from(duration),
+            };
+
             // add message to the database
 
             if context.sql.execute(
-                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?);",
+                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id, ephemeral_timer, ephemeral_timestamp) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?);",
                         paramsv![
                             new_rfc724_mid,
                             self.id,
@@ -956,6 +986,8 @@ impl Chat {
                             new_in_reply_to,
                             new_references,
                             location_id as i32,
+                            ephemeral_timer,
+                            ephemeral_timestamp
                         ]
                     ).await.is_ok() {
                         msg_id = context.sql.get_rowid(
@@ -974,6 +1006,7 @@ impl Chat {
         } else {
             error!(context, "Cannot send message, not configured.",);
         }
+        schedule_ephemeral_task(context).await;
 
         Ok(MsgId::new(msg_id))
     }
@@ -1070,6 +1103,9 @@ pub struct ChatInfo {
     ///
     /// The exact time its muted can be found out via the `chat.mute_duration` property
     pub is_muted: bool,
+
+    /// Ephemeral message timer.
+    pub ephemeral_timer: EphemeralTimer,
     // ToDo:
     // - [ ] deaddrop,
     // - [ ] summary,
@@ -1110,7 +1146,7 @@ pub async fn create_by_msg_id(context: &Context, msg_id: MsgId) -> Result<ChatId
         chat.id.unblock(context).await;
 
         // Sending with 0s as data since multiple messages may have changed.
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -1152,7 +1188,7 @@ pub async fn create_by_contact_id(context: &Context, contact_id: u32) -> Result<
         }
     };
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -1319,7 +1355,7 @@ pub async fn prepare_msg(
 
     msg.state = MessageState::OutPreparing;
     let msg_id = prepare_msg_common(context, chat_id, msg).await?;
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: msg.chat_id,
         msg_id: msg.id,
     });
@@ -1338,11 +1374,12 @@ pub(crate) fn msgtype_has_file(msgtype: Viewtype) -> bool {
         Viewtype::Voice => true,
         Viewtype::Video => true,
         Viewtype::File => true,
+        Viewtype::VideochatInvitation => false,
     }
 }
 
 async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Error> {
-    if msg.viewtype == Viewtype::Text {
+    if msg.viewtype == Viewtype::Text || msg.viewtype == Viewtype::VideochatInvitation {
         // the caller should check if the message text is empty
     } else if msgtype_has_file(msg.viewtype) {
         let blob = msg
@@ -1493,7 +1530,7 @@ pub async fn send_msg_sync(
 
         match status {
             job::Status::Finished(Ok(_)) => {
-                context.emit_event(Event::MsgsChanged {
+                context.emit_event(EventType::MsgsChanged {
                     chat_id: msg.chat_id,
                     msg_id: msg.id,
                 });
@@ -1521,13 +1558,13 @@ async fn send_msg_inner(
     if let Some(send_job) = prepare_send_msg(context, chat_id, msg).await? {
         job::add(context, send_job).await;
 
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: msg.chat_id,
             msg_id: msg.id,
         });
 
         if msg.param.exists(Param::SetLatitude) {
-            context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         }
     }
 
@@ -1575,17 +1612,54 @@ pub async fn send_text_msg(
     send_msg(context, chat_id, &mut msg).await
 }
 
+pub async fn send_videochat_invitation(context: &Context, chat_id: ChatId) -> Result<MsgId, Error> {
+    ensure!(
+        !chat_id.is_special(),
+        "video chat invitation cannot be sent to special chat: {}",
+        chat_id
+    );
+
+    let instance = if let Some(instance) = context.get_config(Config::WebrtcInstance).await {
+        if !instance.is_empty() {
+            instance
+        } else {
+            bail!("webrtc_instance is empty");
+        }
+    } else {
+        bail!("webrtc_instance not set");
+    };
+
+    let instance = Message::create_webrtc_instance(&instance, &dc_create_id());
+
+    let mut msg = Message::new(Viewtype::VideochatInvitation);
+    msg.param.set(Param::WebrtcRoom, &instance);
+    msg.text = Some(
+        context
+            .stock_string_repl_str(
+                StockMessage::VideochatInviteMsgBody,
+                Message::parse_webrtc_instance(&instance).1,
+            )
+            .await,
+    );
+    send_msg(context, chat_id, &mut msg).await
+}
+
 pub async fn get_chat_msgs(
     context: &Context,
     chat_id: ChatId,
     flags: u32,
     marker1before: Option<MsgId>,
-) -> Vec<MsgId> {
-    match delete_device_expired_messages(context).await {
+) -> Vec<ChatItem> {
+    match delete_expired_messages(context).await {
         Err(err) => warn!(context, "Failed to delete expired messages: {}", err),
         Ok(messages_deleted) => {
             if messages_deleted {
-                context.emit_event(Event::MsgsChanged {
+                // Trigger reload of chatlist.
+                //
+                // On desktop chatlist is always shown on the side,
+                // and it is important to update the last message shown
+                // there.
+                context.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 })
@@ -1603,18 +1677,20 @@ pub async fn get_chat_msgs(
             let (curr_id, ts) = row?;
             if let Some(marker_id) = marker1before {
                 if curr_id == marker_id {
-                    ret.push(MsgId::new(DC_MSG_ID_MARKER1));
+                    ret.push(ChatItem::Marker1);
                 }
             }
             if (flags & DC_GCM_ADDDAYMARKER) != 0 {
                 let curr_local_timestamp = ts + cnv_to_local;
                 let curr_day = curr_local_timestamp / 86400;
                 if curr_day != last_day {
-                    ret.push(MsgId::new(DC_MSG_ID_DAYMARKER));
+                    ret.push(ChatItem::DayMarker {
+                        timestamp: curr_day,
+                    });
                     last_day = curr_day;
                 }
             }
-            ret.push(curr_id);
+            ret.push(ChatItem::Message { msg_id: curr_id });
         }
         Ok(ret)
     };
@@ -1706,7 +1782,7 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<(),
         )
         .await?;
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -1738,7 +1814,7 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
         )
         .await?;
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         msg_id: MsgId::new(0),
         chat_id: ChatId::new(0),
     });
@@ -1746,52 +1822,6 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-/// Deletes messages which are expired according to "delete_device_after" setting.
-///
-/// Returns true if any message is deleted, so event can be emitted. If nothing
-/// has been deleted, returns false.
-pub async fn delete_device_expired_messages(context: &Context) -> Result<bool, Error> {
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
-        let threshold_timestamp = time() - delete_device_after;
-
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .await
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .await
-            .unwrap_or_default()
-            .0;
-
-        // Delete expired messages
-        //
-        // Only update the rows that have to be updated, to avoid emitting
-        // unnecessary "chat modified" events.
-        let rows_modified = context
-            .sql
-            .execute(
-                "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
-             WHERE timestamp < ? \
-             AND chat_id > ? \
-             AND chat_id != ? \
-             AND chat_id != ?",
-                paramsv![
-                    DC_CHAT_ID_TRASH,
-                    threshold_timestamp,
-                    DC_CHAT_ID_LAST_SPECIAL,
-                    self_chat_id,
-                    device_chat_id
-                ],
-            )
-            .await?;
-
-        Ok(rows_modified > 0)
-    } else {
-        Ok(false)
-    }
-}
-
 pub async fn get_chat_media(
     context: &Context,
     chat_id: ChatId,
@@ -1955,7 +1985,7 @@ pub async fn create_group_chat(
         chat_id.set_draft_raw(context, &mut draft_msg).await;
     }
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         msg_id: MsgId::new(0),
         chat_id: ChatId::new(0),
     });
@@ -2055,7 +2085,9 @@ pub(crate) async fn add_contact_to_chat_ex(
         /* we should respect this - whatever we send to the group, it gets discarded anyway! */
         emit_event!(
             context,
-            Event::ErrorSelfNotInGroup("Cannot add contact to group; self not in group.".into())
+            EventType::ErrorSelfNotInGroup(
+                "Cannot add contact to group; self not in group.".into()
+            )
         );
         bail!("can not add contact because our account is not part of it");
     }
@@ -2113,7 +2145,7 @@ pub(crate) async fn add_contact_to_chat_ex(
         msg.param.set_int(Param::Arg2, from_handshake.into());
         msg.id = send_msg(context, chat_id, &mut msg).await?;
     }
-    context.emit_event(Event::ChatModified(chat_id));
+    context.emit_event(EventType::ChatModified(chat_id));
     Ok(true)
 }
 
@@ -2275,7 +2307,7 @@ pub async fn set_muted(
         .await
         .is_ok()
     {
-        context.emit_event(Event::ChatModified(chat_id));
+        context.emit_event(EventType::ChatModified(chat_id));
     } else {
         bail!("Failed to set mute duration, chat might not exist -");
     }
@@ -2307,7 +2339,7 @@ pub async fn remove_contact_from_chat(
             if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
                 emit_event!(
                     context,
-                    Event::ErrorSelfNotInGroup(
+                    EventType::ErrorSelfNotInGroup(
                         "Cannot remove contact from chat; self not in group.".into()
                     )
                 );
@@ -2355,7 +2387,7 @@ pub async fn remove_contact_from_chat(
                 // removed it first, it would complicate the
                 // check/encryption logic.
                 success = remove_from_chat_contacts_table(context, chat_id, contact_id).await;
-                context.emit_event(Event::ChatModified(chat_id));
+                context.emit_event(EventType::ChatModified(chat_id));
             }
         }
     }
@@ -2415,7 +2447,7 @@ pub async fn set_chat_name(
         } else if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
             emit_event!(
                 context,
-                Event::ErrorSelfNotInGroup("Cannot set chat name; self not in group".into())
+                EventType::ErrorSelfNotInGroup("Cannot set chat name; self not in group".into())
             );
         } else {
             /* we should respect this - whatever we send to the group, it gets discarded anyway! */
@@ -2445,12 +2477,12 @@ pub async fn set_chat_name(
                         msg.param.set(Param::Arg, &chat.name);
                     }
                     msg.id = send_msg(context, chat_id, &mut msg).await?;
-                    context.emit_event(Event::MsgsChanged {
+                    context.emit_event(EventType::MsgsChanged {
                         chat_id,
                         msg_id: msg.id,
                     });
                 }
-                context.emit_event(Event::ChatModified(chat_id));
+                context.emit_event(EventType::ChatModified(chat_id));
                 success = true;
             }
         }
@@ -2483,7 +2515,9 @@ pub async fn set_chat_profile_image(
     if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
         emit_event!(
             context,
-            Event::ErrorSelfNotInGroup("Cannot set chat profile image; self not in group.".into())
+            EventType::ErrorSelfNotInGroup(
+                "Cannot set chat profile image; self not in group.".into()
+            )
         );
         bail!("Failed to set profile image");
     }
@@ -2522,13 +2556,13 @@ pub async fn set_chat_profile_image(
         msg.id = send_msg(context, chat_id, &mut msg).await?;
         emit_event!(
             context,
-            Event::MsgsChanged {
+            EventType::MsgsChanged {
                 chat_id,
                 msg_id: msg.id
             }
         );
     }
-    emit_event!(context, Event::ChatModified(chat_id));
+    emit_event!(context, EventType::ChatModified(chat_id));
     Ok(())
 }
 
@@ -2612,7 +2646,7 @@ pub async fn forward_msgs(
         }
     }
     for (chat_id, msg_id) in created_chats.iter().zip(created_msgs.iter()) {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: *chat_id,
             msg_id: *msg_id,
         });
@@ -2672,10 +2706,12 @@ pub(crate) async fn get_chat_id_by_grpid(
 /// Adds a message to device chat.
 ///
 /// Optional `label` can be provided to ensure that message is added only once.
-pub async fn add_device_msg(
+/// If `important` is true, a notification will be sent.
+pub async fn add_device_msg_with_importance(
     context: &Context,
     label: Option<&str>,
     msg: Option<&mut Message>,
+    important: bool,
 ) -> Result<MsgId, Error> {
     ensure!(
         label.is_some() || msg.is_some(),
@@ -2735,12 +2771,24 @@ pub async fn add_device_msg(
     }
 
     if !msg_id.is_unset() {
-        context.emit_event(Event::IncomingMsg { chat_id, msg_id });
+        if important {
+            context.emit_event(EventType::IncomingMsg { chat_id, msg_id });
+        } else {
+            context.emit_event(EventType::MsgsChanged { chat_id, msg_id });
+        }
     }
 
     Ok(msg_id)
 }
 
+pub async fn add_device_msg(
+    context: &Context,
+    label: Option<&str>,
+    msg: Option<&mut Message>,
+) -> Result<MsgId, Error> {
+    add_device_msg_with_importance(context, label, msg, false).await
+}
+
 pub async fn was_device_msg_ever_added(context: &Context, label: &str) -> Result<bool, Error> {
     ensure!(!label.is_empty(), "empty label");
     if let Ok(()) = context
@@ -2783,9 +2831,16 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
 /// For example, it can be a message showing that a member was added to a group.
 pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
+    let ephemeral_timer = match chat_id.get_ephemeral_timer(context).await {
+        Err(e) => {
+            warn!(context, "Could not get timer for info msg: {}", e);
+            return;
+        }
+        Ok(ephemeral_timer) => ephemeral_timer,
+    };
 
     if let Err(e) = context.sql.execute(
-        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid) VALUES (?,?,?, ?,?,?, ?,?);",
+        "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid,ephemeral_timer) VALUES (?,?,?, ?,?,?, ?,?,?);",
         paramsv![
             chat_id,
             DC_CONTACT_ID_INFO,
@@ -2795,6 +2850,7 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
             MessageState::InNoticed,
             text.as_ref().to_string(),
             rfc724_mid,
+            ephemeral_timer
         ]
     ).await {
         warn!(context, "Could not add info msg: {}", e);
@@ -2806,7 +2862,7 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
         .get_rowid(context, "msgs", "rfc724_mid", &rfc724_mid)
         .await
         .unwrap_or_default();
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id,
         msg_id: MsgId::new(row_id),
     });
@@ -2844,7 +2900,8 @@ mod tests {
                 "color": 15895624,
                 "profile_image": "",
                 "draft": "",
-                "is_muted": false
+                "is_muted": false,
+                "ephemeral_timer": "Disabled"
             }
         "#;
 

src/chatlist.rs

@@ -5,6 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
+use crate::ephemeral::delete_expired_messages;
 use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
@@ -76,7 +77,7 @@ impl Chatlist {
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
     ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+    ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -99,7 +100,7 @@ impl Chatlist {
 
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context).await {
+        if let Err(err) = delete_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -328,20 +329,30 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let mut ret = Lot::new();
-
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
             None => {
+                let mut ret = Lot::new();
                 ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return ret;
+                return Lot::new();
             }
         };
 
+        Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
+    }
+
+    pub async fn get_summary2(
+        context: &Context,
+        chat_id: ChatId,
+        lastmsg_id: MsgId,
+        chat: Option<&Chat>,
+    ) -> Lot {
+        let mut ret = Lot::new();
+
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
+        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
@@ -350,7 +361,7 @@ impl Chatlist {
 
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {

src/config.rs

@@ -8,7 +8,7 @@ use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
@@ -117,6 +117,14 @@ pub enum Config {
 
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
+
+    #[strum(props(default = "0"))]
+    /// Whether we send a warning if the password is wrong (set to false when we send a warning
+    /// because we do not want to send a second warning)
+    NotifyAboutWrongPw,
+
+    /// address to webrtc instance to use for videochats
+    WebrtcInstance,
 }
 
 impl Context {
@@ -218,7 +226,7 @@ impl Context {
             Config::DeleteDeviceAfter => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
-                self.emit_event(Event::MsgsChanged {
+                self.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 });

src/configure/mod.rs

@@ -1,12 +1,10 @@
 //! Email accounts autoconfiguration process module
 
-#![forbid(clippy::indexing_slicing)]
-
 mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
 
-use anyhow::{bail, ensure, format_err, Context as _, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
@@ -30,7 +28,7 @@ macro_rules! progress {
             $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::ConfigureProgress($progress));
+        $context.emit_event($crate::events::EventType::ConfigureProgress($progress));
     };
 }
 
@@ -72,6 +70,7 @@ impl Context {
 
         let mut param = LoginParam::from_database(self, "").await;
         let success = configure(self, &mut param).await;
+        self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
         if let Some(provider) = provider::get_provider_info(&param.addr) {
             if let Some(config_defaults) = &provider.config_defaults {
@@ -102,11 +101,12 @@ impl Context {
 
         match success {
             Ok(_) => {
+                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+                    .await?;
                 progress!(self, 1000);
                 Ok(())
             }
             Err(err) => {
-                error!(self, "Configure Failed: {}", err);
                 progress!(self, 0);
                 Err(err)
             }
@@ -192,74 +192,26 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     param.server_flags |= keep_flags;
 
     // Step 3: Fill missing fields with defaults
-    if param.mail_server.is_empty() {
-        param.mail_server = format!("imap.{}", param_domain,)
-    }
-    if param.mail_port == 0 {
-        param.mail_port = if 0 != param.server_flags & (0x100 | 0x400) {
-            143
-        } else {
-            993
-        }
-    }
-    if param.mail_user.is_empty() {
-        param.mail_user = param.addr.clone();
-    }
-    if param.send_server.is_empty() && !param.mail_server.is_empty() {
-        param.send_server = param.mail_server.clone();
-        if param.send_server.starts_with("imap.") {
-            param.send_server = param.send_server.replacen("imap", "smtp", 1);
-        }
-    }
-    if param.send_port == 0 {
-        param.send_port = if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
-            587
-        } else if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
-            25
-        } else {
-            465
-        }
-    }
-    if param.send_user.is_empty() && !param.mail_user.is_empty() {
+    if param.send_user.is_empty() {
         param.send_user = param.mail_user.clone();
     }
-    if param.send_pw.is_empty() && !param.mail_pw.is_empty() {
+    if param.send_pw.is_empty() {
         param.send_pw = param.mail_pw.clone()
     }
+    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+    }
+    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+    }
     if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
         param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
         param.server_flags |= DC_LP_AUTH_NORMAL as i32
     }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= if param.send_port == 143 {
-            DC_LP_IMAP_SOCKET_STARTTLS as i32
-        } else {
-            DC_LP_IMAP_SOCKET_SSL as i32
-        }
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= if param.send_port == 587 {
-            DC_LP_SMTP_SOCKET_STARTTLS as i32
-        } else if param.send_port == 25 {
-            DC_LP_SMTP_SOCKET_PLAIN as i32
-        } else {
-            DC_LP_SMTP_SOCKET_SSL as i32
-        }
-    }
 
     // do we have a complete configuration?
     ensure!(
-        !param.mail_server.is_empty()
-            && param.mail_port != 0
-            && !param.mail_user.is_empty()
-            && !param.mail_pw.is_empty()
-            && !param.send_server.is_empty()
-            && param.send_port != 0
-            && !param.send_user.is_empty()
-            && !param.send_pw.is_empty()
-            && param.server_flags != 0,
+        !param.mail_pw.is_empty() && !param.send_pw.is_empty(),
         "Account settings incomplete."
     );
 
@@ -269,10 +221,23 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     let (_s, r) = async_std::sync::channel(1);
     let mut imap = Imap::new(r);
 
-    try_imap_connections(ctx, param, param_autoconfig.is_some(), &mut imap).await?;
-    progress!(ctx, 800);
+    if param_autoconfig.is_some() {
+        if try_imap_one_param(ctx, &param, &mut imap).await.is_err() {
+            bail!("IMAP autoconfig did not succeed");
+        }
+    } else {
+        *param = try_imap_hostnames(ctx, param.clone(), &mut imap).await?;
+    }
+    progress!(ctx, 750);
 
-    try_smtp_connections(ctx, param, param_autoconfig.is_some()).await?;
+    let mut smtp = Smtp::new();
+    if param_autoconfig.is_some() {
+        if try_smtp_one_param(ctx, &param, &mut smtp).await.is_err() {
+            bail!("SMTP autoconfig did not succeed");
+        }
+    } else {
+        *param = try_smtp_hostnames(ctx, param.clone(), &mut smtp).await?;
+    }
     progress!(ctx, 900);
 
     let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
@@ -411,7 +376,7 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
                         p.mail_server = imap.hostname.to_string();
                         p.mail_user = imap.apply_username_pattern(param.addr.clone());
                         p.mail_port = imap.port as i32;
-                        p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.imap_certificate_checks = CertificateChecks::Automatic;
                         p.server_flags |= match imap.socket {
                             provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
                             provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
@@ -420,7 +385,7 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
                         p.send_server = smtp.hostname.to_string();
                         p.send_user = smtp.apply_username_pattern(param.addr.clone());
                         p.send_port = smtp.port as i32;
-                        p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
+                        p.smtp_certificate_checks = CertificateChecks::Automatic;
                         p.server_flags |= match smtp.socket {
                             provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
                             provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
@@ -443,77 +408,105 @@ fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<Login
     None
 }
 
-async fn try_imap_connections(
+async fn try_imap_hostnames(
     context: &Context,
-    param: &mut LoginParam,
-    was_autoconfig: bool,
+    mut param: LoginParam,
     imap: &mut Imap,
-) -> Result<()> {
-    // manually_set_param is used to check whether a particular setting was set manually by the user.
-    // If yes, we do not want to change it to avoid confusing error messages
-    // (you set port 443, but the app tells you it couldn't connect on port 993).
-    let manually_set_param = LoginParam::from_database(context, "").await;
+) -> Result<LoginParam> {
+    if param.mail_server.is_empty() {
+        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
+        let param_domain = parsed.domain;
 
-    // progress 650 and 660
-    if try_imap_connection(context, param, &manually_set_param, was_autoconfig, 0, imap)
-        .await
-        .is_ok()
-    {
-        return Ok(()); // we directly return here if it was autoconfig or the connection succeeded
-    }
+        param.mail_server = param_domain.clone();
+        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
+            return Ok(param);
+        }
 
-    progress!(context, 670);
-    // try_imap_connection() changed the flags and port. Change them back:
-    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-        param.server_flags |= DC_LP_IMAP_SOCKET_SSL;
+        progress!(context, 650);
+        param.mail_server = "imap.".to_string() + &param_domain;
+        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
+            return Ok(param);
+        }
+
+        progress!(context, 700);
+        param.mail_server = "mail.".to_string() + &param_domain;
+        try_imap_ports(context, param, imap).await
+    } else {
+        progress!(context, 700);
+        try_imap_ports(context, param, imap).await
     }
-    if manually_set_param.mail_port == 0 {
-        param.mail_port = 993;
-    }
-    if let Some(at) = param.mail_user.find('@') {
-        param.mail_user = param.mail_user.split_at(at).0.to_string();
-    }
-    if let Some(at) = param.send_user.find('@') {
-        param.send_user = param.send_user.split_at(at).0.to_string();
-    }
-    // progress 680 and 690
-    try_imap_connection(context, param, &manually_set_param, was_autoconfig, 1, imap).await
 }
 
-async fn try_imap_connection(
+// Try various IMAP ports and corresponding TLS settings.
+async fn try_imap_ports(
     context: &Context,
-    param: &mut LoginParam,
-    manually_set_param: &LoginParam,
-    was_autoconfig: bool,
-    variation: usize,
+    mut param: LoginParam,
     imap: &mut Imap,
-) -> Result<()> {
-    if try_imap_one_param(context, param, imap).await.is_ok() {
-        return Ok(());
-    }
-    if was_autoconfig {
-        return Ok(());
-    }
-
-    progress!(context, 650 + variation * 30);
-
-    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS;
-
-        if try_imap_one_param(context, &param, imap).await.is_ok() {
-            return Ok(());
+) -> Result<LoginParam> {
+    // Try to infer port from socket security.
+    if param.mail_port == 0 {
+        if 0 != param.server_flags & DC_LP_IMAP_SOCKET_SSL {
+            param.mail_port = 993
+        }
+        if 0 != param.server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN) {
+            param.mail_port = 143
         }
     }
 
-    progress!(context, 660 + variation * 30);
+    if param.mail_port == 0 {
+        // Neither port nor security is set.
+        //
+        // Try common secure combinations.
 
-    if manually_set_param.mail_port == 0 {
+        // Try TLS over port 993
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
+        param.mail_port = 993;
+        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
+            return Ok(login_param);
+        }
+
+        // Try STARTTLS over port 143
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
         param.mail_port = 143;
-        try_imap_one_param(context, param, imap).await
+        try_imap_usernames(context, param, imap).await
+    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
+        // Try TLS over user-provided port.
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
+        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
+            return Ok(login_param);
+        }
+
+        // Try STARTTLS over user-provided port.
+        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
+        try_imap_usernames(context, param, imap).await
     } else {
-        Err(format_err!("no more possible configs"))
+        try_imap_usernames(context, param, imap).await
+    }
+}
+
+async fn try_imap_usernames(
+    context: &Context,
+    mut param: LoginParam,
+    imap: &mut Imap,
+) -> Result<LoginParam> {
+    if param.mail_user.is_empty() {
+        param.mail_user = param.addr.clone();
+        if let Err(e) = try_imap_one_param(context, &param, imap).await {
+            if let Some(at) = param.mail_user.find('@') {
+                param.mail_user = param.mail_user.split_at(at).0.to_string();
+                try_imap_one_param(context, &param, imap).await?;
+            } else {
+                return Err(e);
+            }
+        }
+        Ok(param)
+    } else {
+        try_imap_one_param(context, &param, imap).await?;
+        Ok(param)
     }
 }
 
@@ -536,48 +529,109 @@ async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Im
     bail!("Could not connect: {}", inf);
 }
 
-async fn try_smtp_connections(
+async fn try_smtp_hostnames(
     context: &Context,
-    param: &mut LoginParam,
-    was_autoconfig: bool,
-) -> Result<()> {
-    // manually_set_param is used to check whether a particular setting was set manually by the user.
-    // If yes, we do not want to change it to avoid confusing error messages
-    // (you set port 443, but the app tells you it couldn't connect on port 993).
-    let manually_set_param = LoginParam::from_database(context, "").await;
+    mut param: LoginParam,
+    smtp: &mut Smtp,
+) -> Result<LoginParam> {
+    if param.send_server.is_empty() {
+        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
+        let param_domain = parsed.domain;
 
-    let mut smtp = Smtp::new();
-    // try to connect to SMTP - if we did not got an autoconfig, the first try was SSL-465 and we do
-    // a second try with STARTTLS-587
-    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
-        return Ok(());
-    }
-    if was_autoconfig {
-        return Ok(());
-    }
-    progress!(context, 850);
+        param.send_server = param_domain.clone();
+        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
+            return Ok(param);
+        }
 
-    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        progress!(context, 800);
+        param.send_server = "smtp.".to_string() + &param_domain;
+        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
+            return Ok(param);
+        }
+
+        progress!(context, 850);
+        param.send_server = "mail.".to_string() + &param_domain;
+        try_smtp_ports(context, param, smtp).await
+    } else {
+        progress!(context, 850);
+        try_smtp_ports(context, param, smtp).await
+    }
+}
+
+// Try various SMTP ports and corresponding TLS settings.
+async fn try_smtp_ports(
+    context: &Context,
+    mut param: LoginParam,
+    smtp: &mut Smtp,
+) -> Result<LoginParam> {
+    // Try to infer port from socket security.
+    if param.send_port == 0 {
+        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
+            param.send_port = 587;
+        }
+        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
+            param.send_port = 25;
+        }
+        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_SSL as i32 {
+            param.send_port = 465;
+        }
+    }
+
+    if param.send_port == 0 {
+        // Neither port nor security is set.
+        //
+        // Try common secure combinations.
+
+        // Try TLS over port 465.
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
+        param.send_port = 465;
+        if let Ok(login_param) = try_smtp_usernames(context, param.clone(), smtp).await {
+            return Ok(login_param);
+        }
+
+        // Try STARTTLS over port 587.
         param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
         param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    }
-    if manually_set_param.send_port == 0 {
         param.send_port = 587;
-    }
+        try_smtp_usernames(context, param, smtp).await
+    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
+        // Try TLS over user-provided port.
+        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
+        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
+        if let Ok(param) = try_smtp_usernames(context, param.clone(), smtp).await {
+            return Ok(param);
+        }
 
-    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
-        return Ok(());
-    }
-    progress!(context, 860);
-
-    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
+        // Try STARTTLS over user-provided port.
         param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
         param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
+        try_smtp_usernames(context, param, smtp).await
+    } else {
+        try_smtp_usernames(context, param, smtp).await
     }
-    if manually_set_param.send_port == 0 {
-        param.send_port = 25;
+}
+
+async fn try_smtp_usernames(
+    context: &Context,
+    mut param: LoginParam,
+    smtp: &mut Smtp,
+) -> Result<LoginParam> {
+    if param.send_user.is_empty() {
+        param.send_user = param.addr.clone();
+        if let Err(e) = try_smtp_one_param(context, &param, smtp).await {
+            if let Some(at) = param.send_user.find('@') {
+                param.send_user = param.send_user.split_at(at).0.to_string();
+                try_smtp_one_param(context, &param, smtp).await?;
+            } else {
+                return Err(e);
+            }
+        }
+        Ok(param)
+    } else {
+        try_smtp_one_param(context, &param, smtp).await?;
+        Ok(param)
     }
-    try_smtp_one_param(context, param, &mut smtp).await
 }
 
 async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Smtp) -> Result<()> {
@@ -652,5 +706,14 @@ mod tests {
         let found_params = get_offline_autoconfig(&context, &params).unwrap();
         assert_eq!(found_params.mail_server, "imap.nauta.cu".to_string());
         assert_eq!(found_params.send_server, "smtp.nauta.cu".to_string());
+
+        assert_eq!(
+            found_params.imap_certificate_checks,
+            CertificateChecks::Automatic
+        );
+        assert_eq!(
+            found_params.smtp_certificate_checks,
+            CertificateChecks::Automatic
+        );
     }
 }

src/constants.rs

@@ -84,6 +84,20 @@ impl Default for KeyGenType {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(i8)]
+pub enum VideochatType {
+    Unknown = 0,
+    BasicWebrtc = 1,
+    Jitsi = 2,
+}
+
+impl Default for VideochatType {
+    fn default() -> Self {
+        VideochatType::Unknown
+    }
+}
+
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
@@ -296,6 +310,9 @@ pub enum Viewtype {
     /// The file is set via dc_msg_set_file()
     /// and retrieved via dc_msg_get_file().
     File = 60,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation = 70,
 }
 
 impl Default for Viewtype {
@@ -323,54 +340,6 @@ const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
 const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
 const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
 const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
-
-// TODO: Strings need some doumentation about used placeholders.
-// These constants are used to set stock translation strings
-
-const DC_STR_NOMESSAGES: usize = 1;
-const DC_STR_SELF: usize = 2;
-const DC_STR_DRAFT: usize = 3;
-const DC_STR_VOICEMESSAGE: usize = 7;
-const DC_STR_DEADDROP: usize = 8;
-const DC_STR_IMAGE: usize = 9;
-const DC_STR_VIDEO: usize = 10;
-const DC_STR_AUDIO: usize = 11;
-const DC_STR_FILE: usize = 12;
-const DC_STR_STATUSLINE: usize = 13;
-const DC_STR_NEWGROUPDRAFT: usize = 14;
-const DC_STR_MSGGRPNAME: usize = 15;
-const DC_STR_MSGGRPIMGCHANGED: usize = 16;
-const DC_STR_MSGADDMEMBER: usize = 17;
-const DC_STR_MSGDELMEMBER: usize = 18;
-const DC_STR_MSGGROUPLEFT: usize = 19;
-const DC_STR_GIF: usize = 23;
-const DC_STR_ENCRYPTEDMSG: usize = 24;
-const DC_STR_E2E_AVAILABLE: usize = 25;
-const DC_STR_ENCR_TRANSP: usize = 27;
-const DC_STR_ENCR_NONE: usize = 28;
-const DC_STR_CANTDECRYPT_MSG_BODY: usize = 29;
-const DC_STR_FINGERPRINTS: usize = 30;
-const DC_STR_READRCPT: usize = 31;
-const DC_STR_READRCPT_MAILBODY: usize = 32;
-const DC_STR_MSGGRPIMGDELETED: usize = 33;
-const DC_STR_E2E_PREFERRED: usize = 34;
-const DC_STR_CONTACT_VERIFIED: usize = 35;
-const DC_STR_CONTACT_NOT_VERIFIED: usize = 36;
-const DC_STR_CONTACT_SETUP_CHANGED: usize = 37;
-const DC_STR_ARCHIVEDCHATS: usize = 40;
-const DC_STR_STARREDMSGS: usize = 41;
-const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
-const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_CANNOT_LOGIN: usize = 60;
-const DC_STR_SERVER_RESPONSE: usize = 61;
-const DC_STR_MSGACTIONBYUSER: usize = 62;
-const DC_STR_MSGACTIONBYME: usize = 63;
-const DC_STR_MSGLOCATIONENABLED: usize = 64;
-const DC_STR_MSGLOCATIONDISABLED: usize = 65;
-const DC_STR_LOCATION: usize = 66;
-const DC_STR_STICKER: usize = 67;
-const DC_STR_COUNT: usize = 67;
 
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 

src/contact.rs

@@ -1,7 +1,5 @@
 //! Contacts module
 
-#![forbid(clippy::indexing_slicing)]
-
 use async_std::path::PathBuf;
 use deltachat_derive::*;
 use itertools::Itertools;
@@ -15,7 +13,7 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
@@ -247,7 +245,7 @@ impl Contact {
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
-        context.emit_event(Event::ContactsChanged(
+        context.emit_event(EventType::ContactsChanged(
             if sth_modified == Modifier::Created {
                 Some(contact_id)
             } else {
@@ -275,7 +273,7 @@ impl Contact {
             .await
             .is_ok()
         {
-            context.emit_event(Event::MsgsChanged {
+            context.emit_event(EventType::MsgsChanged {
                 chat_id: ChatId::new(0),
                 msg_id: MsgId::new(0),
             });
@@ -535,7 +533,7 @@ impl Contact {
             }
         }
         if modify_cnt > 0 {
-            context.emit_event(Event::ContactsChanged(None));
+            context.emit_event(EventType::ContactsChanged(None));
         }
 
         Ok(modify_cnt)
@@ -680,7 +678,7 @@ impl Contact {
         let mut ret = String::new();
 
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let peerstate = Peerstate::from_addr(context, &contact.addr).await;
+            let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
             let loginparam = LoginParam::from_database(context, "configured_").await;
 
             if peerstate.is_some()
@@ -786,7 +784,7 @@ impl Contact {
                 .await
             {
                 Ok(_) => {
-                    context.emit_event(Event::ContactsChanged(None));
+                    context.emit_event(EventType::ContactsChanged(None));
                     return Ok(());
                 }
                 Err(err) => {
@@ -941,7 +939,17 @@ impl Contact {
             }
         }
 
-        let peerstate = Peerstate::from_addr(context, &self.addr).await;
+        let peerstate = match Peerstate::from_addr(context, &self.addr).await {
+            Ok(peerstate) => peerstate,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Failed to load peerstate for address {}: {}", self.addr, err
+                );
+                return VerifiedStatus::Unverified;
+            }
+        };
+
         if let Some(ps) = peerstate {
             if ps.verified_key.is_some() {
                 return VerifiedStatus::BidirectVerified;
@@ -1085,33 +1093,57 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
                     paramsv![new_blocking, 100, contact_id as i32],
                 ).await.is_ok() {
                     Contact::mark_noticed(context, contact_id).await;
-                    context.emit_event(Event::ContactsChanged(None));
+                    context.emit_event(EventType::ContactsChanged(None));
                 }
         }
     }
 }
 
+/// Set profile image for a contact.
+///
+/// The given profile image is expected to be already in the blob directory
+/// as profile images can be set only by receiving messages, this should be always the case, however.
+///
+/// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
+/// this typically happens if we see message with our own profile image, sent from another device.
 pub(crate) async fn set_profile_image(
     context: &Context,
     contact_id: u32,
     profile_image: &AvatarAction,
+    was_encrypted: bool,
 ) -> Result<()> {
-    // the given profile image is expected to be already in the blob directory
-    // as profile images can be set only by receiving messages, this should be always the case, however.
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            contact.param.set(Param::ProfileImage, profile_image);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context
+                        .set_config(Config::Selfavatar, Some(profile_image))
+                        .await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar.");
+                }
+            } else {
+                contact.param.set(Param::ProfileImage, profile_image);
+            }
             true
         }
         AvatarAction::Delete => {
-            contact.param.remove(Param::ProfileImage);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context.set_config(Config::Selfavatar, None).await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar deletion.");
+                }
+            } else {
+                contact.param.remove(Param::ProfileImage);
+            }
             true
         }
     };
     if changed {
         contact.update_param(context).await?;
-        context.emit_event(Event::ContactsChanged(Some(contact_id)));
+        context.emit_event(EventType::ContactsChanged(Some(contact_id)));
     }
     Ok(())
 }

src/context.rs

@@ -6,6 +6,7 @@ use std::ops::Deref;
 
 use async_std::path::{Path, PathBuf};
 use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use async_std::task;
 
 use crate::chat::*;
 use crate::config::Config;
@@ -13,13 +14,11 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::dc_tools::duration_to_str;
 use crate::error::*;
-use crate::events::{Event, EventEmitter, Events};
-use crate::job::{self, Action};
+use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message, MessengerMessage, MsgId};
-use crate::param::Params;
+use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
 use crate::sql::Sql;
 use std::time::SystemTime;
@@ -52,10 +51,16 @@ pub struct InnerContext {
     pub(crate) generating_key_mutex: Mutex<()>,
     /// Mutex to enforce only a single running oauth2 is running.
     pub(crate) oauth2_mutex: Mutex<()>,
+    /// Mutex to prevent a race condition when a "your pw is wrong" warning is sent, resulting in multiple messeges being sent.
+    pub(crate) wrong_pw_warning_mutex: Mutex<()>,
     pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
+
+    /// Id for this context on the current device.
+    pub(crate) id: u32,
 
     creation_time: SystemTime,
 }
@@ -84,7 +89,7 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context.
-    pub async fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
+    pub async fn new(os_name: String, dbfile: PathBuf, id: u32) -> Result<Context> {
         // pretty_env_logger::try_init_timed().ok();
 
         let mut blob_fname = OsString::new();
@@ -94,13 +99,14 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(os_name, dbfile, blobdir).await
+        Context::with_blobdir(os_name, dbfile, blobdir, id).await
     }
 
-    pub async fn with_blobdir(
+    pub(crate) async fn with_blobdir(
         os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
+        id: u32,
     ) -> Result<Context> {
         ensure!(
             blobdir.is_dir().await,
@@ -109,6 +115,7 @@ impl Context {
         );
 
         let inner = InnerContext {
+            id,
             blobdir,
             dbfile,
             os_name: Some(os_name),
@@ -118,9 +125,11 @@ impl Context {
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
+            wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
         };
 
@@ -184,8 +193,11 @@ impl Context {
     }
 
     /// Emits a single event.
-    pub fn emit_event(&self, event: Event) {
-        self.events.emit(event);
+    pub fn emit_event(&self, event: EventType) {
+        self.events.emit(Event {
+            id: self.id,
+            typ: event,
+        });
     }
 
     /// Get the next queued event.
@@ -193,6 +205,11 @@ impl Context {
         self.events.get_emitter()
     }
 
+    /// Get the ID of this context.
+    pub fn get_id(&self) -> u32 {
+        self.id
+    }
+
     // Ongoing process allocation/free/check
 
     pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
@@ -455,32 +472,11 @@ impl Context {
             == Some(folder_name.as_ref().to_string())
     }
 
-    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove).await {
-            return;
-        }
-
-        if self.is_mvbox(folder).await {
-            return;
-        }
-        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
-            if msg.is_setupmessage() {
-                // do not move setup messages;
-                // there may be a non-delta device that wants to handle it
-                return;
-            }
-
-            match msg.is_dc_message {
-                MessengerMessage::No => {}
-                MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job::add(
-                        self,
-                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
-                    )
-                    .await;
-                }
-            }
-        }
+    pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+        let mut blob_fname = OsString::new();
+        blob_fname.push(dbfile.file_name().unwrap_or_default());
+        blob_fname.push("-blobs");
+        dbfile.with_file_name(blob_fname)
     }
 }
 
@@ -534,7 +530,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new("FakeOs".into(), dbfile.into()).await;
+        let res = Context::new("FakeOs".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -549,7 +545,9 @@ mod tests {
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -560,7 +558,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new("FakeOS".into(), dbfile.into()).await;
+        let res = Context::new("FakeOS".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -570,7 +568,9 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -580,7 +580,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -589,7 +589,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 

src/dc_receive_imf.rs

@@ -10,8 +10,9 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer};
 use crate::error::{bail, ensure, format_err, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
 use crate::message::{self, MessageState, MessengerMessage, MsgId};
@@ -92,8 +93,8 @@ pub async fn dc_receive_imf(
         if let Some(create_event_to_send) = create_event_to_send {
             for (chat_id, msg_id) in created_db_entries {
                 let event = match create_event_to_send {
-                    CreateEvent::MsgsChanged => Event::MsgsChanged { msg_id, chat_id },
-                    CreateEvent::IncomingMsg => Event::IncomingMsg { msg_id, chat_id },
+                    CreateEvent::MsgsChanged => EventType::MsgsChanged { msg_id, chat_id },
+                    CreateEvent::IncomingMsg => EventType::IncomingMsg { msg_id, chat_id },
                 };
                 context.emit_event(event);
             }
@@ -196,9 +197,16 @@ pub async fn dc_receive_imf(
     }
 
     if let Some(avatar_action) = &mime_parser.user_avatar {
-        match contact::set_profile_image(&context, from_id, avatar_action).await {
+        match contact::set_profile_image(
+            &context,
+            from_id,
+            avatar_action,
+            mime_parser.was_encrypted(),
+        )
+        .await
+        {
             Ok(()) => {
-                context.emit_event(Event::ChatModified(chat_id));
+                context.emit_event(EventType::ChatModified(chat_id));
             }
             Err(err) => {
                 warn!(context, "reveive_imf cannot update profile image: {}", err);
@@ -223,24 +231,29 @@ pub async fn dc_receive_imf(
                 )
                 .await;
             }
-        } else {
+        } else if insert_msg_id
+            .needs_move(context, server_folder.as_ref())
+            .await
+            .unwrap_or_default()
+        {
             // Move message if we don't delete it immediately.
-            context
-                .do_heuristics_moves(server_folder.as_ref(), insert_msg_id)
-                .await;
-            if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
-                // This is a Delta Chat MDN. Mark as read.
-                job::add(
-                    context,
-                    job::Job::new(
-                        Action::MarkseenMsgOnImap,
-                        insert_msg_id.to_u32(),
-                        Params::new(),
-                        0,
-                    ),
-                )
-                .await;
-            }
+            job::add(
+                context,
+                job::Job::new(Action::MoveMsg, insert_msg_id.to_u32(), Params::new(), 0),
+            )
+            .await;
+        } else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
+            // This is a Delta Chat MDN. Mark as read.
+            job::add(
+                context,
+                job::Job::new(
+                    Action::MarkseenMsgOnImap,
+                    insert_msg_id.to_u32(),
+                    Params::new(),
+                    0,
+                ),
+            )
+            .await;
         }
     }
 
@@ -619,16 +632,75 @@ async fn add_parts(
             *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
         }
     }
+
+    // Extract ephemeral timer from the message.
+    let mut ephemeral_timer = if let Some(value) = mime_parser.get(HeaderDef::EphemeralTimer) {
+        match value.parse::<EphemeralTimer>() {
+            Ok(timer) => timer,
+            Err(err) => {
+                warn!(
+                    context,
+                    "can't parse ephemeral timer \"{}\": {}", value, err
+                );
+                EphemeralTimer::Disabled
+            }
+        }
+    } else {
+        EphemeralTimer::Disabled
+    };
+
+    let location_kml_is = mime_parser.location_kml.is_some();
+    let is_mdn = !mime_parser.mdn_reports.is_empty();
+
+    // Apply ephemeral timer changes to the chat.
+    //
+    // Only non-hidden timers are applied now. Timers from hidden
+    // messages such as read receipts can be useful to detect
+    // ephemeral timer support, but timer changes without visible
+    // received messages may be confusing to the user.
+    if !*hidden
+        && !location_kml_is
+        && !is_mdn
+        && (*chat_id).get_ephemeral_timer(context).await? != ephemeral_timer
+    {
+        if let Err(err) = (*chat_id)
+            .inner_set_ephemeral_timer(context, ephemeral_timer)
+            .await
+        {
+            warn!(
+                context,
+                "failed to modify timer for chat {}: {}", chat_id, err
+            );
+        } else if mime_parser.is_system_message != SystemMessage::EphemeralTimerChanged {
+            chat::add_info_msg(
+                context,
+                *chat_id,
+                stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
+            )
+            .await;
+        }
+    }
+
+    if mime_parser.is_system_message == SystemMessage::EphemeralTimerChanged {
+        set_better_msg(
+            mime_parser,
+            stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
+        );
+
+        // Do not delete the system message itself.
+        //
+        // This prevents confusion when timer is changed
+        // to 1 week, and then changed to 1 hour: after 1
+        // hour, only the message about the change to 1
+        // week is left.
+        ephemeral_timer = EphemeralTimer::Disabled;
+    }
+
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
+    let in_fresh = state == MessageState::InFresh;
     let rcvd_timestamp = time();
-    let sort_timestamp = calc_sort_timestamp(
-        context,
-        *sent_timestamp,
-        *chat_id,
-        state == MessageState::InFresh,
-    )
-    .await;
+    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, in_fresh).await;
     *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
@@ -655,7 +727,6 @@ async fn add_parts(
 
     let mut parts = std::mem::replace(&mut mime_parser.parts, Vec::new());
     let server_folder = server_folder.as_ref().to_string();
-    let location_kml_is = mime_parser.location_kml.is_some();
     let is_system_message = mime_parser.is_system_message;
     let mime_headers = if save_mime_headers {
         Some(String::from_utf8_lossy(imf_raw).to_string())
@@ -665,7 +736,6 @@ async fn add_parts(
     let sent_timestamp = *sent_timestamp;
     let is_hidden = *hidden;
     let chat_id = *chat_id;
-    let is_mdn = !mime_parser.mdn_reports.is_empty();
 
     // TODO: can this clone be avoided?
     let rfc724_mid = rfc724_mid.to_string();
@@ -682,8 +752,8 @@ async fn add_parts(
                     "INSERT INTO msgs \
          (rfc724_mid, server_folder, server_uid, chat_id, from_id, to_id, timestamp, \
          timestamp_sent, timestamp_rcvd, type, state, msgrmsg,  txt, txt_raw, param, \
-         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references, error) \
-         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?, ?);",
+         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references, error, ephemeral_timer, ephemeral_timestamp) \
+         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?, ?,?,?);",
                 )?;
 
                 let is_location_kml = location_kml_is
@@ -705,6 +775,15 @@ async fn add_parts(
                     part.param.set_int(Param::Cmd, is_system_message as i32);
                 }
 
+                let ephemeral_timestamp = if in_fresh {
+                    0
+                } else {
+                    match ephemeral_timer {
+                        EphemeralTimer::Disabled => 0,
+                        EphemeralTimer::Enabled { duration } => rcvd_timestamp + i64::from(duration)
+                    }
+                };
+
                 stmt.execute(paramsv![
                     rfc724_mid,
                     server_folder,
@@ -728,6 +807,8 @@ async fn add_parts(
                     mime_in_reply_to,
                     mime_references,
                     part.error,
+                    ephemeral_timer,
+                    ephemeral_timestamp
                 ])?;
 
                 drop(stmt);
@@ -783,15 +864,17 @@ async fn add_parts(
         chat.update_param(context).await?;
         Ok(())
     }
-    update_last_subject(context, chat_id, mime_parser)
-        .await
-        .unwrap_or_else(|e| {
-            warn!(
-                context,
-                "Could not update LastSubject of chat: {}",
-                e.to_string()
-            )
-        });
+    if !is_mdn {
+        update_last_subject(context, chat_id, mime_parser)
+            .await
+            .unwrap_or_else(|e| {
+                warn!(
+                    context,
+                    "Could not update LastSubject of chat: {}",
+                    e.to_string()
+                )
+            });
+    }
 
     Ok(())
 }
@@ -852,7 +935,7 @@ async fn save_locations(
         }
     }
     if send_event {
-        context.emit_event(Event::LocationChanged(Some(from_id)));
+        context.emit_event(EventType::LocationChanged(Some(from_id)));
     }
 }
 
@@ -1113,23 +1196,19 @@ async fn create_or_lookup_group(
 
     // again, check chat_id
     if chat_id.is_special() {
-        return if group_explicitly_left {
-            Ok((ChatId::new(DC_CHAT_ID_TRASH), chat_id_blocked))
+        if mime_parser.decrypting_failed {
+            // It is possible that the message was sent to a valid,
+            // yet unknown group, which was rejected because
+            // Chat-Group-Name, which is in the encrypted part, was
+            // not found. We can't create a properly named group in
+            // this case, so assign error message to 1:1 chat with the
+            // sender instead.
+            return Ok((ChatId::new(0), Blocked::Not));
         } else {
-            create_or_lookup_adhoc_group(
-                context,
-                mime_parser,
-                allow_creation,
-                create_blocked,
-                from_id,
-                to_ids,
-            )
-            .await
-            .map_err(|err| {
-                warn!(context, "failed to create ad-hoc group: {:?}", err);
-                err
-            })
-        };
+            // The message was decrypted successfully, but contains a late "quit" or otherwise
+            // unwanted message.
+            return Ok((ChatId::new(DC_CHAT_ID_TRASH), chat_id_blocked));
+        }
     }
 
     // We have a valid chat_id > DC_CHAT_ID_LAST_SPECIAL.
@@ -1160,7 +1239,7 @@ async fn create_or_lookup_group(
                     .await
                     .is_ok()
                 {
-                    context.emit_event(Event::ChatModified(chat_id));
+                    context.emit_event(EventType::ChatModified(chat_id));
                 }
             }
         }
@@ -1219,7 +1298,7 @@ async fn create_or_lookup_group(
     }
 
     if send_EVENT_CHAT_MODIFIED {
-        context.emit_event(Event::ChatModified(chat_id));
+        context.emit_event(EventType::ChatModified(chat_id));
     }
     Ok((chat_id, chat_id_blocked))
 }
@@ -1314,12 +1393,10 @@ async fn create_or_lookup_adhoc_group(
         // decrypted.
         //
         // The subject may be encrypted and contain a placeholder such
-        // as "...". Besides that, it is possible that the message was
-        // sent to a valid, yet unknown group, which was rejected
-        // because Chat-Group-Name, which is in the encrypted part,
-        // was not found. Generating a new ID in this case would
-        // result in creation of a twin group with a different group
-        // ID.
+        // as "...". It can also be a COI group, with encrypted
+        // Chat-Group-ID and incompatible Message-ID format.
+        //
+        // Instead, assign the message to 1:1 chat with the sender.
         warn!(
             context,
             "not creating ad-hoc group for message that cannot be decrypted"
@@ -1358,7 +1435,7 @@ async fn create_or_lookup_adhoc_group(
         chat::add_to_chat_contacts_table(context, new_chat_id, member_id).await;
     }
 
-    context.emit_event(Event::ChatModified(new_chat_id));
+    context.emit_event(EventType::ChatModified(new_chat_id));
 
     Ok((new_chat_id, create_blocked))
 }
@@ -1451,6 +1528,7 @@ async fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
     hex_hash(&members)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn hex_hash(s: impl AsRef<str>) -> String {
     let bytes = s.as_ref().as_bytes();
     let result = Sha256::digest(bytes);
@@ -1503,7 +1581,7 @@ async fn search_chat_ids_by_contact_ids(
                             matches = 0;
                             mismatches = 0;
                         }
-                        if matches < contact_ids.len() && contact_id == contact_ids[matches] {
+                        if contact_ids.get(matches) == Some(&contact_id) {
                             matches += 1;
                         } else {
                             mismatches += 1;
@@ -1537,7 +1615,7 @@ async fn check_verified_properties(
     // this check is skipped for SELF as there is no proper SELF-peerstate
     // and results in group-splits otherwise.
     if from_id != DC_CONTACT_ID_SELF {
-        let peerstate = Peerstate::from_addr(context, contact.get_addr()).await;
+        let peerstate = Peerstate::from_addr(context, contact.get_addr()).await?;
 
         if peerstate.is_none()
             || contact.is_verified_ex(context, peerstate.as_ref()).await
@@ -1591,7 +1669,7 @@ async fn check_verified_properties(
             context.is_self_addr(&to_addr).await
         );
         let mut is_verified = _is_verified != 0;
-        let peerstate = Peerstate::from_addr(context, &to_addr).await;
+        let peerstate = Peerstate::from_addr(context, &to_addr).await?;
 
         // mark gossiped keys (if any) as verified
         if mimeparser.gossipped_addr.contains(&to_addr) {
@@ -1631,10 +1709,11 @@ async fn check_verified_properties(
 
 fn set_better_msg(mime_parser: &mut MimeMessage, better_msg: impl AsRef<str>) {
     let msg = better_msg.as_ref();
-    if !msg.is_empty() && !mime_parser.parts.is_empty() {
-        let part = &mut mime_parser.parts[0];
-        if part.typ == Viewtype::Text {
-            part.msg = msg.to_string();
+    if !msg.is_empty() {
+        if let Some(part) = mime_parser.parts.get_mut(0) {
+            if part.typ == Viewtype::Text {
+                part.msg = msg.to_string();
+            }
         }
     }
 }
@@ -1793,7 +1872,7 @@ fn dc_create_incoming_rfc724_mid(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat::ChatVisibility;
+    use crate::chat::{ChatItem, ChatVisibility};
     use crate::chatlist::Chatlist;
     use crate::message::Message;
     use crate::test_utils::*;
@@ -2118,7 +2197,11 @@ mod tests {
         .unwrap();
         let msgs = chat::get_chat_msgs(&t.ctx, group_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
         let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
             .await
             .unwrap();
@@ -2253,7 +2336,11 @@ mod tests {
         );
         let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
         let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
             .await
             .unwrap();
@@ -2385,7 +2472,7 @@ mod tests {
             "haeclirth.sinoenrat@yahoo.com",
             "1680295672.3657931.1591783872936@mail.yahoo.com",
             include_bytes!("../test-data/message/yahoo_ndn.eml"),
-            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com"
+            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com [...]"
         )
         .await;
     }
@@ -2409,7 +2496,7 @@ mod tests {
             "snaerituhaeirns@gmail.com",
             "9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de",
             include_bytes!("../test-data/message/gmx_ndn.eml"),
-            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp"
+            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp [...]"
         )
         .await;
     }
@@ -2516,9 +2603,12 @@ mod tests {
         assert_eq!(msg.state, MessageState::OutFailed);
 
         let msgs = chat::get_chat_msgs(&t.ctx, msg.chat_id, 0, None).await;
-        let last_msg = Message::load_from_db(&t.ctx, *msgs.last().unwrap())
-            .await
-            .unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
+        let last_msg = Message::load_from_db(&t.ctx, *msg_id).await.unwrap();
 
         assert_eq!(
             last_msg.text,

src/dc_tools.rs

@@ -4,6 +4,7 @@
 use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
+use std::io::Cursor;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
@@ -14,7 +15,7 @@ use rand::{thread_rng, Rng};
 
 use crate::context::Context;
 use crate::error::{bail, Error};
-use crate::events::Event;
+use crate::events::EventType;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
     0 != v && 0 == v & (v - 1)
@@ -22,6 +23,7 @@ pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let ellipse = "[...]";
 
@@ -54,6 +56,7 @@ const COLORS: [u32; 16] = [
     0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
 ];
 
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let str_lower = s.as_ref().to_lowercase();
     let mut checksum = 0;
@@ -198,7 +201,7 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
 pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str) -> String {
     let hostname = from_addr
         .find('@')
-        .map(|k| &from_addr[k..])
+        .and_then(|k| from_addr.get(k..))
         .unwrap_or("@nohost");
     match grpid {
         Some(grpid) => format!("Gr.{}.{}{}", grpid, dc_create_id(), hostname),
@@ -240,9 +243,9 @@ pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
 
 /// Returns the `(width, height)` of the given image buffer.
 pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
-    let meta = image_meta::load_from_buf(buf)?;
-
-    Ok((meta.dimensions.width, meta.dimensions.height))
+    let image = image::io::Reader::new(Cursor::new(buf)).with_guessed_format()?;
+    let dimensions = image.into_dimensions()?;
+    Ok(dimensions)
 }
 
 /// Expand paths relative to $BLOBDIR into absolute paths.
@@ -283,7 +286,7 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     let dpath = format!("{}", path.as_ref().to_string_lossy());
     match fs::remove_file(path_abs).await {
         Ok(_) => {
-            context.emit_event(Event::DeletedBlobFile(dpath));
+            context.emit_event(EventType::DeletedBlobFile(dpath));
             true
         }
         Err(err) => {
@@ -447,7 +450,7 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn dc_get_next_backup_path(
+pub(crate) async fn get_next_backup_path_old(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -467,6 +470,32 @@ pub(crate) async fn dc_get_next_backup_path(
     bail!("could not create backup file, disk full?");
 }
 
+/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+pub(crate) async fn get_next_backup_path_new(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf), Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
@@ -913,4 +942,22 @@ mod tests {
             "3h 1m 0s"
         );
     }
+
+    #[test]
+    fn test_get_filemeta() {
+        let data = include_bytes!("../test-data/image/avatar900x900.png");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 900);
+        assert_eq!(h, 900);
+
+        let data = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 1000);
+        assert_eq!(h, 1000);
+
+        let data = include_bytes!("../test-data/image/image100x50.gif");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 100);
+        assert_eq!(h, 50);
+    }
 }

src/dehtml.rs

@@ -46,6 +46,12 @@ pub fn dehtml(buf: &str) -> String {
             Ok(quick_xml::events::Event::End(ref e)) => dehtml_endtag_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::Text(ref e)) => dehtml_text_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::CData(ref e)) => dehtml_cdata_cb(e, &mut dehtml),
+            Ok(quick_xml::events::Event::Empty(ref e)) => {
+                // Handle empty tags as a start tag immediately followed by end tag.
+                // For example, `<p/>` is treated as `<p></p>`.
+                dehtml_starttag_cb(e, &mut dehtml, &reader);
+                dehtml_endtag_cb(&BytesEnd::borrowed(e.name()), &mut dehtml);
+            }
             Err(e) => {
                 eprintln!(
                     "Parse html error: Error at position {}: {:?}",
@@ -182,6 +188,10 @@ mod tests {
             ("&amp; bar", "& bar"),
             // Note missing '
             ("<a href='/foo.png>Hi</a> ", ""),
+            (
+                "<a href='https://get.delta.chat/'/>",
+                "[](https://get.delta.chat/)",
+            ),
             ("", ""),
         ];
         for (input, output) in cases {
@@ -191,10 +201,10 @@ mod tests {
 
     #[test]
     fn test_dehtml_parse_br() {
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2<br/>line3\n\r";
         let plain = dehtml(html);
 
-        assert_eq!(plain, "line1\n\r\r\rline2");
+        assert_eq!(plain, "line1\n\r\r\rline2\nline3");
     }
 
     #[test]

src/e2ee.rs

@@ -15,7 +15,6 @@ use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
-use crate::securejoin::handle_degrade_event;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
@@ -115,6 +114,14 @@ impl EncryptHelper {
     }
 }
 
+/// Tries to decrypt a message, but only if it is structured as an
+/// Autocrypt message.
+///
+/// Returns decrypted body and a set of valid signature fingerprints
+/// if successful.
+///
+/// If the message is wrongly signed, this will still return the decrypted
+/// message but the HashSet will be empty.
 pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
@@ -132,7 +139,7 @@ pub async fn try_decrypt(
     let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
 
     if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &from).await;
+        peerstate = Peerstate::from_addr(context, &from).await?;
 
         if let Some(ref mut peerstate) = peerstate {
             if let Some(ref header) = autocryptheader {
@@ -155,17 +162,14 @@ pub async fn try_decrypt(
     let mut signatures = HashSet::default();
 
     if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-        peerstate = Peerstate::from_addr(&context, &from).await;
+        peerstate = Peerstate::from_addr(&context, &from).await?;
     }
     if let Some(peerstate) = peerstate {
-        if peerstate.degrade_event.is_some() {
-            handle_degrade_event(context, &peerstate).await?;
-        }
-        if let Some(key) = peerstate.gossip_key {
-            public_keyring_for_validate.add(key);
-        }
+        peerstate.handle_fingerprint_change(context).await?;
         if let Some(key) = peerstate.public_key {
             public_keyring_for_validate.add(key);
+        } else if let Some(key) = peerstate.gossip_key {
+            public_keyring_for_validate.add(key);
         }
     }
 
@@ -187,24 +191,23 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
         "Not a multipart/encrypted message: {}",
         mail.ctype.mimetype
     );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
+    if let [first_part, second_part] = &mail.subparts[..] {
+        ensure!(
+            first_part.ctype.mimetype == "application/pgp-encrypted",
+            "Invalid Autocrypt Level 1 version part: {:?}",
+            first_part.ctype,
+        );
 
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
+        ensure!(
+            second_part.ctype.mimetype == "application/octet-stream",
+            "Invalid Autocrypt Level 1 encrypted part: {:?}",
+            second_part.ctype
+        );
 
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
-    );
-
-    Ok(&mail.subparts[1])
+        Ok(second_part)
+    } else {
+        bail!("Invalid Autocrypt Level 1 Mime Parts")
+    }
 }
 
 async fn decrypt_if_autocrypt_message<'a>(
@@ -214,13 +217,6 @@ async fn decrypt_if_autocrypt_message<'a>(
     public_keyring_for_validate: Keyring<SignedPublicKey>,
     ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
-    //  The returned bool is true if we detected an Autocrypt-encrypted
-    // message and successfully decrypted it. Decryption then modifies the
-    // passed in mime structure in place. The returned bool is false
-    // if it was not an Autocrypt message.
-    //
-    // Errors are returned for failures related to decryption of AC-messages.
-
     let encrypted_data_part = match get_autocrypt_mime(mail) {
         Err(_) => {
             // not an autocrypt mime message, abort and ignore
@@ -260,13 +256,16 @@ async fn decrypt_part(
         )
         .await?;
 
-        ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
+        // If the message was wrongly or not signed, still return the plain text.
+        // The caller has to check the signatures then.
+
         return Ok(Some(plain));
     }
 
     Ok(None)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     if let Some(index) = input.iter().position(|b| *b > b' ') {
         if input.len() - index > 26 {

src/ephemeral.rs

@@ -0,0 +1,528 @@
+//! # Ephemeral messages
+//!
+//! Ephemeral messages are messages that have an Ephemeral-Timer
+//! header attached to them, which specifies time in seconds after
+//! which the message should be deleted both from the device and from
+//! the server. The timer is started when the message is marked as
+//! seen, which usually happens when its contents is displayed on
+//! device screen.
+//!
+//! Each chat, including 1:1, group chats and "saved messages" chat,
+//! has its own ephemeral timer setting, which is applied to all
+//! messages sent to the chat. The setting is synchronized to all the
+//! devices participating in the chat by applying the timer value from
+//! all received messages, including BCC-self ones, to the chat. This
+//! way the setting is eventually synchronized among all participants.
+//!
+//! When user changes ephemeral timer setting for the chat, a system
+//! message is automatically sent to update the setting for all
+//! participants. This allows changing the setting for a chat like any
+//! group chat setting, e.g. name and avatar, without the need to
+//! write an actual message.
+//!
+//! ## Device settings
+//!
+//! In addition to per-chat ephemeral message setting, each device has
+//! two global user-configured settings that complement per-chat
+//! settings: `delete_device_after` and `delete_server_after`. These
+//! settings are not synchronized among devices and apply to all
+//! messages known to the device, including messages sent or received
+//! before configuring the setting.
+//!
+//! `delete_device_after` configures the maximum time device is
+//! storing the messages locally. `delete_server_after` configures the
+//! time after which device will delete the messages it knows about
+//! from the server.
+//!
+//! ## How messages are deleted
+//!
+//! When the message is deleted locally, its contents is removed and
+//! it is moved to the trash chat. This database entry is then used to
+//! track the Message-ID and corresponding IMAP folder and UID until
+//! the message is deleted from the server. Vice versa, when device
+//! deletes the message from the server, it removes IMAP folder and
+//! UID information, but keeps the message contents. When database
+//! entry is both moved to trash chat and does not contain UID
+//! information, it is deleted from the database, leaving no trace of
+//! the message.
+//!
+//! ## When messages are deleted
+//!
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
+//!
+//! Server deletion happens by generating IMAP deletion jobs based on
+//! the database entries which are expired either according to their
+//! ephemeral message timers or global `delete_server_after` setting.
+
+use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use crate::constants::{
+    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::error::{ensure, Error};
+use crate::events::EventType;
+use crate::message::{Message, MessageState, MsgId};
+use crate::mimeparser::SystemMessage;
+use crate::sql;
+use crate::stock::StockMessage;
+use async_std::task;
+use serde::{Deserialize, Serialize};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
+pub enum Timer {
+    Disabled,
+    Enabled { duration: u32 },
+}
+
+impl Timer {
+    pub fn to_u32(self) -> u32 {
+        match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => duration,
+        }
+    }
+
+    pub fn from_u32(duration: u32) -> Self {
+        if duration == 0 {
+            Self::Disabled
+        } else {
+            Self::Enabled { duration }
+        }
+    }
+}
+
+impl Default for Timer {
+    fn default() -> Self {
+        Self::Disabled
+    }
+}
+
+impl ToString for Timer {
+    fn to_string(&self) -> String {
+        self.to_u32().to_string()
+    }
+}
+
+impl FromStr for Timer {
+    type Err = ParseIntError;
+
+    fn from_str(input: &str) -> Result<Timer, ParseIntError> {
+        input.parse::<u32>().map(Self::from_u32)
+    }
+}
+
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
+    }
+}
+
+impl ChatId {
+    /// Get ephemeral message timer value in seconds.
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+        let timer = context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
+            )
+            .await?;
+        Ok(timer.unwrap_or_default())
+    }
+
+    /// Set ephemeral timer value without sending a message.
+    ///
+    /// Used when a message arrives indicating that someone else has
+    /// changed the timer value for a chat.
+    pub(crate) async fn inner_set_ephemeral_timer(
+        self,
+        context: &Context,
+        timer: Timer,
+    ) -> Result<(), Error> {
+        ensure!(!self.is_special(), "Invalid chat ID");
+
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+             SET ephemeral_timer=?
+             WHERE id=?;",
+                paramsv![timer, self],
+            )
+            .await?;
+
+        context.emit_event(EventType::ChatEphemeralTimerModified {
+            chat_id: self,
+            timer,
+        });
+        Ok(())
+    }
+
+    /// Set ephemeral message timer value in seconds.
+    ///
+    /// If timer value is 0, disable ephemeral message timer.
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+        if timer == self.get_ephemeral_timer(context).await? {
+            return Ok(());
+        }
+        self.inner_set_ephemeral_timer(context, timer).await?;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
+        if let Err(err) = send_msg(context, self, &mut msg).await {
+            error!(
+                context,
+                "Failed to send a message about ephemeral message timer change: {:?}", err
+            );
+        }
+        Ok(())
+    }
+}
+
+/// Returns a stock message saying that ephemeral timer is changed to `timer` by `from_id`.
+pub(crate) async fn stock_ephemeral_timer_changed(
+    context: &Context,
+    timer: Timer,
+    from_id: u32,
+) -> String {
+    let stock_message = match timer {
+        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+        Timer::Enabled { duration } => match duration {
+            60 => StockMessage::MsgEphemeralTimerMinute,
+            3600 => StockMessage::MsgEphemeralTimerHour,
+            86400 => StockMessage::MsgEphemeralTimerDay,
+            604_800 => StockMessage::MsgEphemeralTimerWeek,
+            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
+            _ => StockMessage::MsgEphemeralTimerEnabled,
+        },
+    };
+
+    context
+        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
+        .await
+}
+
+impl MsgId {
+    /// Returns ephemeral message timer value for the message.
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
+        let res = match context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
+            )
+            .await?
+        {
+            None | Some(0) => Timer::Disabled,
+            Some(duration) => Timer::Enabled { duration },
+        };
+        Ok(res)
+    }
+
+    /// Starts ephemeral message timer for the message if it is not started yet.
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
+        if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
+            let ephemeral_timestamp = time() + i64::from(duration);
+
+            context
+                .sql
+                .execute(
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
+                WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
+                AND id = ?",
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
+                )
+                .await?;
+            schedule_ephemeral_task(context).await;
+        }
+        Ok(())
+    }
+}
+
+/// Deletes messages which are expired according to
+/// `delete_device_after` setting or `ephemeral_timestamp` column.
+///
+/// Returns true if any message is deleted, so caller can emit
+/// MsgsChanged event. If nothing has been deleted, returns
+/// false. This function does not emit the MsgsChanged event itself,
+/// because it is also called when chatlist is reloaded, and emitting
+/// MsgsChanged there will cause infinite reload loop.
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+    let mut updated = context
+        .sql
+        .execute(
+            "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE \
+             ephemeral_timestamp != 0 \
+             AND ephemeral_timestamp < ? \
+             AND chat_id != ?",
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+        )
+        .await?
+        > 0;
+
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
+        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap_or_default()
+            .0;
+        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+            .await
+            .unwrap_or_default()
+            .0;
+
+        let threshold_timestamp = time() - delete_device_after;
+
+        // Delete expired messages
+        //
+        // Only update the rows that have to be updated, to avoid emitting
+        // unnecessary "chat modified" events.
+        let rows_modified = context
+            .sql
+            .execute(
+                "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE timestamp < ? \
+             AND chat_id > ? \
+             AND chat_id != ? \
+             AND chat_id != ?",
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
+            )
+            .await?;
+
+        updated |= rows_modified > 0;
+    }
+
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
+}
+
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
+///
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
+    let ephemeral_timestamp: Option<i64> = match context
+        .sql
+        .query_get_value_result(
+            "SELECT ephemeral_timestamp \
+         FROM msgs \
+         WHERE ephemeral_timestamp != 0 \
+           AND chat_id != ? \
+         ORDER BY ephemeral_timestamp ASC \
+         LIMIT 1",
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
+        )
+        .await
+    {
+        Err(err) => {
+            warn!(context, "Can't calculate next ephemeral timeout: {}", err);
+            return;
+        }
+        Ok(ephemeral_timestamp) => ephemeral_timestamp,
+    };
+
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
+
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+        let now = SystemTime::now();
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
+
+        if let Ok(duration) = until.duration_since(now) {
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                emit_event!(
+                    context1,
+                    EventType::MsgsChanged {
+                        chat_id: ChatId::new(0),
+                        msg_id: MsgId::new(0)
+                    }
+                );
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            emit_event!(
+                context,
+                EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0)
+                }
+            );
+        }
+    }
+}
+
+/// Returns ID of any expired message that should be deleted from the server.
+///
+/// It looks up the trash chat too, to find messages that are already
+/// deleted locally, but not deleted on the server.
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+    let now = time();
+
+    let threshold_timestamp = match context.get_config_delete_server_after().await {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
+
+    context
+        .sql
+        .query_row_optional(
+            "SELECT id FROM msgs \
+         WHERE ( \
+         timestamp < ? \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         ) \
+         AND server_uid != 0 \
+         LIMIT 1",
+            paramsv![threshold_timestamp, now],
+            |row| row.get::<_, MsgId>(0),
+        )
+        .await
+}
+
+/// Start ephemeral timers for seen messages if they are not started
+/// yet.
+///
+/// It is possible that timers are not started due to a missing or
+/// failed `MsgId.start_ephemeral_timer()` call, either in the current
+/// or previous version of Delta Chat.
+///
+/// This function is supposed to be called in the background,
+/// e.g. from housekeeping task.
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+    context
+        .sql
+        .execute(
+            "UPDATE msgs \
+    SET ephemeral_timestamp = ? + ephemeral_timer \
+    WHERE ephemeral_timer > 0 \
+    AND ephemeral_timestamp = 0 \
+    AND state NOT IN (?, ?, ?)",
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
+        )
+        .await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::*;
+
+    #[async_std::test]
+    async fn test_stock_ephemeral_messages() {
+        let context = TestContext::new().await.ctx;
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            "Message deletion timer is disabled by me."
+        );
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
+            "Message deletion timer is disabled."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
+            "Message deletion timer is set to 1 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
+            "Message deletion timer is set to 30 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
+            "Message deletion timer is set to 1 minute."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
+            "Message deletion timer is set to 1 hour."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 day."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 week."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 4 * 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 4 weeks."
+        );
+    }
+}

src/events.rs

@@ -1,10 +1,13 @@
 //! # Events specification
 
+use std::ops::Deref;
+
 use async_std::path::PathBuf;
 use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
 
 #[derive(Debug)]
@@ -53,14 +56,32 @@ impl EventEmitter {
         async_std::task::block_on(self.recv())
     }
 
-    /// Blocking async recv of an event. Return `None` if the `Sender` has been droped.
+    /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
         // TODO: change once we can use async channels internally.
         self.0.recv().await.ok()
     }
+
+    pub fn try_recv(&self) -> Result<Event, async_std::sync::TryRecvError> {
+        self.0.try_recv()
+    }
 }
 
-impl Event {
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Event {
+    pub id: u32,
+    pub typ: EventType,
+}
+
+impl Deref for Event {
+    type Target = EventType;
+
+    fn deref(&self) -> &EventType {
+        &self.typ
+    }
+}
+
+impl EventType {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
@@ -71,7 +92,7 @@ impl Event {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
-pub enum Event {
+pub enum EventType {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
@@ -106,7 +127,7 @@ pub enum Event {
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
-    /// Emitted when an new file in the $BLOBDIR was created
+    /// Emitted when an file in the $BLOBDIR was deleted
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -139,7 +160,6 @@ pub enum Event {
     /// Network errors should be reported to users in a non-disturbing way,
     /// however, as network errors may come in a sequence,
     /// it is not useful to raise each an every error to the user.
-    /// For this purpose, data1 is set to 1 if the error is probably worth reporting.
     ///
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
@@ -189,9 +209,19 @@ pub enum Event {
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// This event does not include ephemeral timer modification, which
+    /// is a separate event.
     #[strum(props(id = "2020"))]
     ChatModified(ChatId),
 
+    /// Chat ephemeral timer changed.
+    #[strum(props(id = "2021"))]
+    ChatEphemeralTimerModified {
+        chat_id: ChatId,
+        timer: EphemeralTimer,
+    },
+
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.

src/headerdef.rs

@@ -35,6 +35,7 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
+    ChatWebrtcRoom,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,
@@ -42,6 +43,7 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    EphemeralTimer,
     _TestHeader,
 }
 

src/imap/client.rs

@@ -56,7 +56,7 @@ impl Client {
     pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
         self,
         auth_type: S,
-        authenticator: &A,
+        authenticator: A,
     ) -> std::result::Result<Session, (ImapError, Self)> {
         let Client { inner, is_secure } = self;
         let session =
@@ -85,9 +85,6 @@ impl Client {
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
 
         let _greeting = client
             .read_response()
@@ -104,9 +101,6 @@ impl Client {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
         let _greeting = client
             .read_response()
             .await

src/imap/idle.rs

@@ -1,6 +1,7 @@
 use super::Imap;
 
 use async_imap::extensions::idle::IdleResponse;
+use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
@@ -13,19 +14,19 @@ type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
+    #[error("IMAP IDLE protocol failed to init/complete: {0}")]
     IdleProtocolFailed(#[from] async_imap::error::Error),
 
-    #[error("IMAP IDLE protocol timed out")]
+    #[error("IMAP IDLE protocol timed out: {0}")]
     IdleTimeout(#[from] async_std::future::TimeoutError),
 
     #[error("IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[error("IMAP select folder error")]
+    #[error("IMAP select folder error: {0}")]
     SelectFolderError(#[from] select_folder::Error),
 
-    #[error("Setup handle error")]
+    #[error("Setup handle error: {0}")]
     SetupHandleError(#[from] super::Error),
 }
 
@@ -48,11 +49,27 @@ impl Imap {
 
         self.select_folder(context, watch_folder.clone()).await?;
 
-        let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
-        if let Some(session) = session {
+        if let Some(session) = self.session.take() {
+            // if we have unsolicited responses we directly return
+            let mut unsolicited_exists = false;
+            while let Ok(response) = session.unsolicited_responses.try_recv() {
+                match response {
+                    UnsolicitedResponse::Exists(_) => {
+                        warn!(context, "skip idle, got unsolicited EXISTS {:?}", response);
+                        unsolicited_exists = true;
+                    }
+                    _ => info!(context, "ignoring unsolicited response {:?}", response),
+                }
+            }
+
+            if unsolicited_exists {
+                self.session = Some(session);
+                return Ok(info);
+            }
+
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
                 return Err(Error::IdleProtocolFailed(err));
@@ -65,68 +82,43 @@ impl Imap {
                 Interrupt(InterruptInfo),
             }
 
-            if self.skip_next_idle_wait {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait = false;
-                drop(idle_wait);
+            info!(context, "Idle entering wait-on-remote state");
+            let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
+                let probe_network = self.idle_interrupt.recv().await;
+
+                // cancel imap idle connection properly
                 drop(interrupt);
 
-                info!(context, "Idle wait was skipped");
-            } else {
-                info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
-                    self.idle_interrupt.recv().map(|probe_network| {
-                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
-                    }),
-                );
+                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+            });
 
-                match fut.await {
-                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
-                        info!(context, "Idle has NewData");
-                    }
-                    // TODO: idle_wait does not distinguish manual interrupts
-                    // from Timeouts if we would know it's a Timeout we could bail
-                    // directly and reconnect .
-                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                        info!(context, "Idle-wait timeout or interruption");
-                    }
-                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Ok(Event::Interrupt(i)) => {
-                        info = i;
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Err(err) => {
-                        warn!(context, "Idle wait errored: {:?}", err);
-                    }
+            match fut.await {
+                Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
+                    info!(context, "Idle has NewData {:?}", x);
+                }
+                Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                    info!(context, "Idle-wait timeout or interruption");
+                }
+                Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                    info!(context, "Idle wait was interrupted");
+                }
+                Ok(Event::Interrupt(i)) => {
+                    info = i;
+                    info!(context, "Idle wait was interrupted");
+                }
+                Err(err) => {
+                    warn!(context, "Idle wait errored: {:?}", err);
                 }
             }
 
-            // if we can't properly terminate the idle
-            // protocol let's break the connection.
-            let res = handle
+            let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(|err| {
-                    self.trigger_reconnect();
-                    Error::IdleTimeout(err)
-                })?;
-
-            match res {
-                Ok(session) => {
-                    self.session = Some(Session { inner: session });
-                }
-                Err(err) => {
-                    // if we cannot terminate IDLE it probably
-                    // means that we waited long (with idle_wait)
-                    // but the network went away/changed
-                    self.trigger_reconnect();
-                    return Err(Error::IdleProtocolFailed(err));
-                }
-            }
+                .map_err(Error::IdleTimeout)??;
+            self.session = Some(Session { inner: session });
+        } else {
+            warn!(context, "Attempted to idle without a session");
         }
 
         Ok(info)
@@ -148,73 +140,66 @@ impl Imap {
             return self.idle_interrupt.recv().await.unwrap_or_default();
         }
 
-        let mut info: InterruptInfo = Default::default();
-        if self.skip_next_idle_wait {
-            // interrupt_idle has happened before we
-            // provided self.interrupt
-            self.skip_next_idle_wait = false;
-            info!(context, "fake-idle wait was skipped");
-        } else {
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        // check every minute if there are new messages
+        // TODO: grow sleep durations / make them more flexible
+        let mut interval = async_std::stream::interval(Duration::from_secs(60));
 
-            enum Event {
-                Tick,
-                Interrupt(InterruptInfo),
-            }
-            // loop until we are interrupted or if we fetched something
-            info =
-                loop {
-                    use futures::future::FutureExt;
-                    match interval
-                        .next()
-                        .map(|_| Event::Tick)
-                        .race(self.idle_interrupt.recv().map(|probe_network| {
-                            Event::Interrupt(probe_network.unwrap_or_default())
-                        }))
-                        .await
-                    {
-                        Event::Tick => {
-                            // try to connect with proper login params
-                            // (setup_handle_if_needed might not know about them if we
-                            // never successfully connected)
-                            if let Err(err) = self.connect_configured(context).await {
-                                warn!(context, "fake_idle: could not connect: {}", err);
-                                continue;
-                            }
-                            if self.config.can_idle {
-                                // we only fake-idled because network was gone during IDLE, probably
-                                break InterruptInfo::new(false, None);
-                            }
-                            info!(context, "fake_idle is connected");
-                            // we are connected, let's see if fetching messages results
-                            // in anything.  If so, we behave as if IDLE had data but
-                            // will have already fetched the messages so perform_*_fetch
-                            // will not find any new.
+        enum Event {
+            Tick,
+            Interrupt(InterruptInfo),
+        }
+        // loop until we are interrupted or if we fetched something
+        let info = loop {
+            use futures::future::FutureExt;
+            match interval
+                .next()
+                .map(|_| Event::Tick)
+                .race(
+                    self.idle_interrupt
+                        .recv()
+                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                )
+                .await
+            {
+                Event::Tick => {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context).await {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break InterruptInfo::new(false, None);
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
 
-                            if let Some(ref watch_folder) = watch_folder {
-                                match self.fetch_new_messages(context, watch_folder).await {
-                                    Ok(res) => {
-                                        info!(context, "fetch_new_messages returned {:?}", res);
-                                        if res {
-                                            break InterruptInfo::new(false, None);
-                                        }
-                                    }
-                                    Err(err) => {
-                                        error!(context, "could not fetch from folder: {}", err);
-                                        self.trigger_reconnect()
-                                    }
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break InterruptInfo::new(false, None);
                                 }
                             }
-                        }
-                        Event::Interrupt(info) => {
-                            // Interrupt
-                            break info;
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
                         }
                     }
-                };
-        }
+                }
+                Event::Interrupt(info) => {
+                    // Interrupt
+                    break info;
+                }
+            }
+        };
 
         info!(
             context,

src/imap/mod.rs

@@ -3,8 +3,6 @@
 //! uses [async-email/async-imap](https://github.com/async-email/async-imap)
 //! to implement connect, fetch, delete functionality with standard IMAP servers.
 
-#![forbid(clippy::indexing_slicing)]
-
 use std::collections::BTreeMap;
 
 use async_imap::{
@@ -21,16 +19,16 @@ use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf, from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list,
 };
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
-use crate::message::{self, update_server_uid};
+use crate::message::{self, update_server_uid, MessageState};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::provider::get_provider_info;
-use crate::{scheduler::InterruptInfo, stock::StockMessage};
+use crate::{chat, scheduler::InterruptInfo, stock::StockMessage};
 
 mod client;
 mod idle;
@@ -38,6 +36,7 @@ pub mod select_folder;
 mod session;
 
 use client::Client;
+use message::Message;
 use session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
@@ -117,8 +116,8 @@ pub struct Imap {
     session: Option<Session>,
     connected: bool,
     interrupt: Option<stop_token::StopSource>,
-    skip_next_idle_wait: bool,
     should_reconnect: bool,
+    login_failed_once: bool,
 }
 
 #[derive(Debug)]
@@ -130,7 +129,7 @@ struct OAuth2 {
 impl async_imap::Authenticator for OAuth2 {
     type Response = String;
 
-    fn process(&self, _data: &[u8]) -> Self::Response {
+    fn process(&mut self, _data: &[u8]) -> Self::Response {
         format!(
             "user={}\x01auth=Bearer {}\x01\x01",
             self.user, self.access_token
@@ -191,8 +190,8 @@ impl Imap {
             session: Default::default(),
             connected: Default::default(),
             interrupt: Default::default(),
-            skip_next_idle_wait: Default::default(),
             should_reconnect: Default::default(),
+            login_failed_once: Default::default(),
         }
     }
 
@@ -263,7 +262,7 @@ impl Imap {
                             user: imap_user.into(),
                             access_token: token,
                         };
-                        client.authenticate("XOAUTH2", &auth).await
+                        client.authenticate("XOAUTH2", auth).await
                     } else {
                         return Err(Error::OauthError);
                     }
@@ -285,7 +284,7 @@ impl Imap {
                         .await
                 };
                 // IMAP connection failures are reported to users
-                emit_event!(context, Event::ErrorNetwork(message));
+                emit_event!(context, EventType::ErrorNetwork(message));
                 return Err(Error::ConnectionFailed(err.to_string()));
             }
         };
@@ -297,18 +296,40 @@ impl Imap {
                 // needs to be set here to ensure it is set on reconnects.
                 self.connected = true;
                 self.session = Some(session);
+                self.login_failed_once = false;
                 Ok(())
             }
+
             Err((err, _)) => {
                 let imap_user = self.config.imap_user.to_owned();
                 let message = context
                     .stock_string_repl_str(StockMessage::CannotLogin, &imap_user)
                     .await;
 
-                emit_event!(
-                    context,
-                    Event::ErrorNetwork(format!("{} ({})", message, err))
-                );
+                warn!(context, "{} ({})", message, err);
+                emit_event!(context, EventType::ErrorNetwork(message.clone()));
+
+                let lock = context.wrong_pw_warning_mutex.lock().await;
+                if self.login_failed_once
+                    && context.get_config_bool(Config::NotifyAboutWrongPw).await
+                {
+                    if let Err(e) = context.set_config(Config::NotifyAboutWrongPw, None).await {
+                        warn!(context, "{}", e);
+                    }
+                    drop(lock);
+
+                    let mut msg = Message::new(Viewtype::Text);
+                    msg.text = Some(message);
+                    if let Err(e) =
+                        chat::add_device_msg_with_importance(context, None, Some(&mut msg), true)
+                            .await
+                    {
+                        warn!(context, "{}", e);
+                    }
+                } else {
+                    self.login_failed_once = true;
+                }
+
                 self.trigger_reconnect();
                 Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
             }
@@ -424,7 +445,7 @@ impl Imap {
                         self.connected = true;
                         emit_event!(
                             context,
-                            Event::ImapConnected(format!(
+                            EventType::ImapConnected(format!(
                                 "IMAP-LOGIN as {}, capabilities: {}",
                                 lp.mail_user, caps_list,
                             ))
@@ -552,8 +573,14 @@ impl Imap {
                     let set = format!("{}", mailbox.exists);
                     match session.fetch(set, JUST_UID).await {
                         Ok(mut list) => {
-                            if let Some(Ok(msg)) = list.next().await {
-                                msg.uid.unwrap_or_default()
+                            let mut new_last_seen_uid = None;
+                            while let Some(fetch) = list.next().await.transpose()? {
+                                if fetch.message == mailbox.exists && fetch.uid.is_some() {
+                                    new_last_seen_uid = fetch.uid;
+                                }
+                            }
+                            if let Some(new_last_seen_uid) = new_last_seen_uid {
+                                new_last_seen_uid
                             } else {
                                 return Err(Error::Other("failed to fetch".into()));
                             }
@@ -783,7 +810,6 @@ impl Imap {
         let mut last_uid = None;
         let mut count = 0;
 
-        let mut tasks = Vec::with_capacity(server_uids.len());
         while let Some(Ok(msg)) = msgs.next().await {
             let server_uid = msg.uid.unwrap_or_default();
 
@@ -803,31 +829,17 @@ impl Imap {
             let context = context.clone();
             let folder = folder.clone();
 
-            let task = async_std::task::spawn(async move {
-                // safe, as we checked above that there is a body.
-                let body = msg.body().unwrap();
-                let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
+            // safe, as we checked above that there is a body.
+            let body = msg.body().unwrap();
+            let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
-                match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
-                    Ok(_) => Some(server_uid),
-                    Err(err) => {
-                        warn!(context, "dc_receive_imf error: {}", err);
-                        None
-                    }
-                }
-            });
-            tasks.push(task);
-        }
-
-        for task in futures::future::join_all(tasks).await {
-            match task {
-                Some(uid) => {
-                    last_uid = Some(uid);
-                }
-                None => {
+            match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
+                Ok(_) => last_uid = Some(server_uid),
+                Err(err) => {
+                    warn!(context, "dc_receive_imf error: {}", err);
                     read_errors += 1;
                 }
-            }
+            };
         }
 
         if count != server_uids.len() {
@@ -876,7 +888,7 @@ impl Imap {
                     Ok(_) => {
                         emit_event!(
                             context,
-                            Event::ImapMessageMoved(format!(
+                            EventType::ImapMessageMoved(format!(
                                 "IMAP Message {} moved to {}",
                                 display_folder_id, dest_folder
                             ))
@@ -920,7 +932,7 @@ impl Imap {
             warn!(context, "Cannot mark {} as \"Deleted\" after copy.", uid);
             emit_event!(
                 context,
-                Event::ImapMessageMoved(format!(
+                EventType::ImapMessageMoved(format!(
                     "IMAP Message {} copied to {} (delete FAILED)",
                     display_folder_id, dest_folder
                 ))
@@ -930,7 +942,7 @@ impl Imap {
             self.config.selected_folder_needs_expunge = true;
             emit_event!(
                 context,
-                Event::ImapMessageMoved(format!(
+                EventType::ImapMessageMoved(format!(
                     "IMAP Message {} copied to {} (delete successfull)",
                     display_folder_id, dest_folder
                 ))
@@ -964,7 +976,11 @@ impl Imap {
         if let Some(ref mut session) = &mut self.session {
             let query = format!("+FLAGS ({})", flag);
             match session.uid_store(uid_set, &query).await {
-                Ok(_) => {}
+                Ok(mut responses) => {
+                    while let Some(_response) = responses.next().await {
+                        // Read all the responses
+                    }
+                }
                 Err(err) => {
                     warn!(
                         context,
@@ -1067,8 +1083,35 @@ impl Imap {
         if let Some(ref mut session) = &mut self.session {
             match session.uid_fetch(set, DELETE_CHECK_FLAGS).await {
                 Ok(mut msgs) => {
-                    let fetch = if let Some(Ok(fetch)) = msgs.next().await {
-                        fetch
+                    let mut remote_message_id = None;
+
+                    while let Some(response) = msgs.next().await {
+                        match response {
+                            Ok(fetch) => {
+                                if fetch.uid == Some(uid) {
+                                    remote_message_id = get_fetch_headers(&fetch)
+                                        .and_then(|headers| prefetch_get_message_id(&headers))
+                                        .ok();
+                                }
+                            }
+                            Err(err) => {
+                                warn!(context, "IMAP fetch error {}", err);
+                                return ImapActionResult::RetryLater;
+                            }
+                        }
+                    }
+
+                    if let Some(remote_message_id) = remote_message_id {
+                        if remote_message_id != message_id {
+                            warn!(
+                                context,
+                                "Cannot delete on IMAP, {}: remote message-id '{}' != '{}'",
+                                display_imap_id,
+                                remote_message_id,
+                                message_id,
+                            );
+                            return ImapActionResult::Failed;
+                        }
                     } else {
                         warn!(
                             context,
@@ -1077,21 +1120,6 @@ impl Imap {
                             message_id,
                         );
                         return ImapActionResult::AlreadyDone;
-                    };
-
-                    let remote_message_id = get_fetch_headers(&fetch)
-                        .and_then(|headers| prefetch_get_message_id(&headers))
-                        .unwrap_or_default();
-
-                    if remote_message_id != message_id {
-                        warn!(
-                            context,
-                            "Cannot delete on IMAP, {}: remote message-id '{}' != '{}'",
-                            display_imap_id,
-                            remote_message_id,
-                            message_id,
-                        );
-                        return ImapActionResult::Failed;
                     }
                 }
                 Err(err) => {
@@ -1114,7 +1142,7 @@ impl Imap {
         } else {
             emit_event!(
                 context,
-                Event::ImapMessageDeleted(format!(
+                EventType::ImapMessageDeleted(format!(
                     "IMAP Message {} marked as deleted [{}]",
                     display_imap_id, message_id
                 ))
@@ -1288,7 +1316,7 @@ impl Imap {
         self.config.selected_folder_needs_expunge = true;
         match self.select_folder::<String>(context, None).await {
             Ok(()) => {
-                emit_event!(context, Event::ImapFolderEmptied(folder.to_string()));
+                emit_event!(context, EventType::ImapFolderEmptied(folder.to_string()));
             }
             Err(err) => {
                 error!(context, "expunge failed {}: {:?}", folder, err);
@@ -1360,14 +1388,26 @@ async fn precheck_imf(
             let delete_server_after = context.get_config_delete_server_after().await;
 
             if delete_server_after != Some(0) {
-                context
-                    .do_heuristics_moves(server_folder.as_ref(), msg_id)
+                if msg_id
+                    .needs_move(context, server_folder)
+                    .await
+                    .unwrap_or_default()
+                {
+                    // If the bcc-self message is not moved, directly
+                    // add MarkSeen job, otherwise MarkSeen job is
+                    // added after the Move Job completed.
+                    job::add(
+                        context,
+                        job::Job::new(Action::MoveMsg, msg_id.to_u32(), Params::new(), 0),
+                    )
                     .await;
-                job::add(
-                    context,
-                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
-                )
-                .await;
+                } else {
+                    job::add(
+                        context,
+                        job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                    )
+                    .await;
+                }
             }
         } else if old_server_folder != server_folder {
             info!(
@@ -1402,6 +1442,13 @@ async fn precheck_imf(
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
             update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
+            if let Ok(MessageState::InSeen) = msg_id.get_state(context).await {
+                job::add(
+                    context,
+                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                )
+                .await;
+            };
             context
                 .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
                 .await;
@@ -1474,7 +1521,6 @@ pub(crate) async fn prefetch_should_download(
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message
-        || maybe_ndn
         || match show_emails {
             ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
             ShowEmails::AcceptedContacts => {
@@ -1482,8 +1528,8 @@ pub(crate) async fn prefetch_should_download(
             }
             ShowEmails::All => true,
         };
-    let show = show && !blocked_contact;
-    Ok(show)
+    let should_download = (show && !blocked_contact) || maybe_ndn;
+    Ok(should_download)
 }
 
 async fn message_needs_processing(

src/imap/select_folder.rs

@@ -51,6 +51,14 @@ impl Imap {
         Ok(())
     }
 
+    /// Issues a CLOSE command if selected folder needs expunge.
+    pub(crate) async fn maybe_close_folder(&mut self, context: &Context) -> Result<()> {
+        if self.config.selected_folder_needs_expunge {
+            self.close_folder(context).await?;
+        }
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -76,10 +84,7 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.selected_folder_needs_expunge };
-        if needs_expunge {
-            self.close_folder(context).await?;
-        }
+        self.maybe_close_folder(context).await?;
 
         // select new folder
         if let Some(ref folder) = folder {

src/imex.rs

@@ -1,10 +1,16 @@
 //! # Import/export module
 
 use std::any::Any;
-use std::cmp::{max, min};
+use std::{
+    cmp::{max, min},
+    ffi::OsStr,
+};
 
 use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
+use async_std::{
+    fs::{self, File},
+    prelude::*,
+};
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -16,7 +22,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -24,6 +30,11 @@ use crate::param::*;
 use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock::StockMessage;
+use async_tar::Archive;
+
+// Name of the database file in the backup.
+const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
+const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 #[repr(i32)]
@@ -42,8 +53,8 @@ pub enum ImexMode {
     /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
-    /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
-    /// the format is `delta-chat.<day>-<number>.bak`
+    /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
+    /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
     /// `param1` is the file (not: directory) to import. The file is normally
@@ -85,6 +96,37 @@ pub async fn imex(
 
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+    let dir_name = dir_name.as_ref();
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut newest_backup_name = "".to_string();
+    let mut newest_backup_path: Option<PathBuf> = None;
+
+    while let Some(dirent) = dir_iter.next().await {
+        if let Ok(dirent) = dirent {
+            let path = dirent.path();
+            let name = dirent.file_name();
+            let name: String = name.to_string_lossy().into();
+            if name.starts_with("delta-chat")
+                && name.ends_with(".tar")
+                && (newest_backup_name.is_empty() || name > newest_backup_name)
+            {
+                // We just use string comparison to determine which backup is newer.
+                // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
+                newest_backup_path = Some(path);
+                newest_backup_name = name;
+            }
+        }
+    }
+
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => has_backup_old(context, dir_name).await,
+        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
+    }
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
@@ -178,10 +220,11 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 ///
 /// The `passphrase` must be at least 2 characters long.
 pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
-    ensure!(
-        passphrase.len() >= 2,
-        "Passphrase must be at least 2 chars long."
-    );
+    let passphrase_begin = if let Some(passphrase_begin) = passphrase.get(..2) {
+        passphrase_begin
+    } else {
+        bail!("Passphrase must be at least 2 chars long.");
+    };
     let private_key = SignedSecretKey::load_self(context).await?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
@@ -196,7 +239,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
             "Passphrase-Format: numeric9x4\r\n",
             "Passphrase-Begin: {}"
         ),
-        &passphrase[..2]
+        passphrase_begin
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
@@ -368,11 +411,11 @@ async fn imex_inner(
     ensure!(param.is_some(), "No Import/export dir/file given.");
 
     info!(context, "Import/export process started.");
-    context.emit_event(Event::ImexProgress(10));
+    context.emit_event(EventType::ImexProgress(10));
 
     ensure!(context.sql.is_open().await, "Database not opened.");
 
-    let path = param.unwrap();
+    let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?;
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -385,18 +428,22 @@ async fn imex_inner(
     let success = match what {
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
-        ImexMode::ExportBackup => export_backup(context, path).await,
+
+        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
+        // (now is 07/2020)
+        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
     };
 
     match success {
         Ok(()) => {
             info!(context, "IMEX successfully completed");
-            context.emit_event(Event::ImexProgress(1000));
+            context.emit_event(EventType::ImexProgress(1000));
             Ok(())
         }
         Err(err) => {
-            context.emit_event(Event::ImexProgress(0));
+            context.emit_event(EventType::ImexProgress(0));
             bail!("IMEX FAILED to complete: {}", err);
         }
     }
@@ -404,6 +451,75 @@ async fn imex_inner(
 
 /// Import Backup
 async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+    if backup_to_import
+        .as_ref()
+        .to_string_lossy()
+        .ends_with(".bak")
+    {
+        // Backwards compability
+        return import_backup_old(context, backup_to_import).await;
+    }
+
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.as_ref().display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !context.is_configured().await,
+        "Cannot import backups to accounts in use."
+    );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
+
+    let backup_file = File::open(backup_to_import).await?;
+    let archive = Archive::new(backup_file);
+    let mut entries = archive.entries()?;
+    while let Some(file) = entries.next().await {
+        let f = &mut file?;
+        if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
+            // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
+            f.unpack_in(context.get_blobdir()).await?;
+            fs::rename(
+                context.get_blobdir().join(DBFILE_BACKUP_NAME),
+                context.get_dbfile(),
+            )
+            .await?;
+            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
+        } else {
+            // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
+            f.unpack_in(context.get_blobdir()).await?;
+            let from_path = context.get_blobdir().join(f.path()?);
+            if from_path.is_file().await {
+                if let Some(name) = from_path.file_name() {
+                    fs::rename(&from_path, context.get_blobdir().join(name)).await?;
+                } else {
+                    warn!(context, "No file name");
+                }
+            }
+        }
+    }
+
+    ensure!(
+        context
+            .sql
+            .open(&context, &context.get_dbfile(), false)
+            .await,
+        "could not re-open db"
+    );
+
+    delete_and_reset_all_device_msgs(&context).await?;
+
+    Ok(())
+}
+
+async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -448,27 +564,33 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
     );
 
-    let files = context
+    // Load IDs only for now, without the file contents, to avoid
+    // consuming too much memory.
+    let file_ids = context
         .sql
         .query_map(
-            "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
+            "SELECT id FROM backup_blobs ORDER BY id",
             paramsv![],
-            |row| {
-                let name: String = row.get(0)?;
-                let blob: Vec<u8> = row.get(1)?;
-
-                Ok((name, blob))
-            },
-            |files| {
-                files
-                    .collect::<std::result::Result<Vec<_>, _>>()
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
                     .map_err(Into::into)
             },
         )
         .await?;
 
     let mut all_files_extracted = true;
-    for (processed_files_cnt, (file_name, file_blob)) in files.into_iter().enumerate() {
+    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        // Load a single blob into memory
+        let (file_name, file_blob) = context
+            .sql
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?)),
+            )
+            .await?;
+
         if context.shall_stop_ongoing().await {
             all_files_extracted = false;
             break;
@@ -480,7 +602,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         if permille > 990 {
             permille = 990
         }
-        context.emit_event(Event::ImexProgress(permille));
+        context.emit_event(EventType::ImexProgress(permille));
         if file_blob.is_empty() {
             continue;
         }
@@ -505,14 +627,83 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
-The macro avoids weird values of 0% or 100% while still working. */
+#[allow(unused)]
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
+    let now = time();
+    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+
+    context
+        .sql
+        .set_raw_config_int(context, "backup_time", now as i32)
+        .await?;
+    sql::housekeeping(context).await;
+
+    context
+        .sql
+        .execute("VACUUM;", paramsv![])
+        .await
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
+
+    // we close the database during the export
+    context.sql.close().await;
+
+    info!(
+        context,
+        "Backup '{}' to '{}'.",
+        context.get_dbfile().display(),
+        dest_path.display(),
+    );
+
+    let res = export_backup_inner(context, &temp_path).await;
+
+    // we re-open the database after export is finished
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await;
+
+    match &res {
+        Ok(_) => {
+            fs::rename(temp_path, &dest_path).await?;
+            context.emit_event(EventType::ImexFileWritten(dest_path));
+        }
+        Err(e) => {
+            error!(context, "backup failed: {}", e);
+            // Not using dc_delete_file() here because it would send a DeletedBlobFile event
+            fs::remove_file(temp_path).await?;
+        }
+    }
+
+    res
+}
+
+async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
+    let file = File::create(temp_path).await?;
+
+    let mut builder = async_tar::Builder::new(file);
+
+    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
+    builder
+        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
+        .await?;
+
+    context.emit_event(EventType::ImexProgress(500));
+
+    builder
+        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
+        .await?;
+
+    builder.finish().await?;
+    Ok(())
+}
+
+async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
+    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
     let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
     sql::housekeeping(context).await;
@@ -556,7 +747,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
             dest_sql
                 .set_raw_config_int(context, "backup_time", now as i32)
                 .await?;
-            context.emit_event(Event::ImexFileWritten(dest_path_filename));
+            context.emit_event(EventType::ImexFileWritten(dest_path_filename));
             Ok(())
         }
     };
@@ -595,7 +786,7 @@ async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
             }
             processed_files_cnt += 1;
             let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-            context.emit_event(Event::ImexProgress(permille));
+            context.emit_event(EventType::ImexProgress(permille));
 
             let name_f = entry.file_name();
             let name = name_f.to_string_lossy();
@@ -762,7 +953,7 @@ where
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
-        context.emit_event(Event::ImexFileWritten(file_name));
+        context.emit_event(EventType::ImexFileWritten(file_name));
     }
     res
 }

src/job.rs

@@ -21,8 +21,9 @@ use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::load_imap_deletion_msgid;
 use crate::error::{bail, ensure, format_err, Error, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::imap::*;
 use crate::location;
 use crate::login_param::LoginParam;
@@ -94,7 +95,6 @@ pub enum Action {
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
     EmptyServer = 107,
-    OldDeleteMsgOnImap = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -123,7 +123,6 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
-            OldDeleteMsgOnImap => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
             EmptyServer => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
@@ -254,40 +253,48 @@ impl Job {
 
                 let res = match err {
                     async_smtp::smtp::error::Error::Permanent(ref response) => {
-                        match response.code {
+                        // Workaround for incorrectly configured servers returning permanent errors
+                        // instead of temporary ones.
+                        let maybe_transient = match response.code {
                             // Sometimes servers send a permanent error when actually it is a temporary error
                             // For documentation see https://tools.ietf.org/html/rfc3463
-
-                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
                             Code {
                                 category: Category::MailSystem,
                                 detail: Detail::Zero,
                                 ..
-                            } => Status::RetryLater,
-
-                            _ => {
-                                // If we do not retry, add an info message to the chat
-                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
-                                match Message::load_from_db(context, MsgId::new(self.foreign_id))
-                                    .await
-                                {
-                                    Ok(message) => {
-                                        chat::add_info_msg(
-                                            context,
-                                            message.chat_id,
-                                            err.to_string(),
-                                        )
-                                        .await
-                                    }
-                                    Err(e) => warn!(
-                                        context,
-                                        "couldn't load chat_id to inform user about SMTP error: {}",
-                                        e
-                                    ),
-                                };
-
-                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } => {
+                                // Ignore status code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                                // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
+                                // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
+                                //
+                                // Other enhanced status codes, such as Postfix
+                                // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
+                                // are not ignored.
+                                response.message.get(0) == Some(&"5.5.0".to_string())
                             }
+                            _ => false,
+                        };
+
+                        if maybe_transient {
+                            Status::RetryLater
+                        } else {
+                            // If we do not retry, add an info message to the chat.
+                            // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
+                            // should definitely go here, because user has to open the link to
+                            // resume message sending.
+                            let msg_id = MsgId::new(self.foreign_id);
+                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+                            match Message::load_from_db(context, msg_id).await {
+                                Ok(message) => {
+                                    chat::add_info_msg(context, message.chat_id, err.to_string())
+                                        .await
+                                }
+                                Err(e) => error!(
+                                    context,
+                                    "couldn't load chat_id to inform user about SMTP error: {}", e
+                                ),
+                            };
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
                     async_smtp::smtp::error::Error::Transient(_) => {
@@ -624,7 +631,9 @@ impl Job {
             }
         }
         if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            imap.empty_folder(context, "INBOX").await;
+            if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
+                imap.empty_folder(context, &inbox_folder).await;
+            }
         }
         Status::Finished(Ok(()))
     }
@@ -638,7 +647,21 @@ impl Job {
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
-        match imap.set_seen(context, folder, msg.server_uid).await {
+
+        let result = if msg.server_uid == 0 {
+            // The message is moved or deleted by us.
+            //
+            // Do not call set_seen with zero UID, as it will return
+            // ImapActionResult::RetryLater, but we do not want to
+            // retry. If the message was moved, we will create another
+            // job to mark the message as seen later. If it was
+            // deleted, there is nothing to do.
+            ImapActionResult::Failed
+        } else {
+            imap.set_seen(context, folder, msg.server_uid).await
+        };
+
+        match result {
             ImapActionResult::RetryLater => Status::RetryLater,
             ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
             ImapActionResult::Success | ImapActionResult::Failed => {
@@ -646,7 +669,18 @@ impl Job {
                 // we want to send out an MDN anyway
                 // The job will not be retried so locally
                 // there is no risk of double-sending MDNs.
+                //
+                // Read receipts for system messages are never
+                // sent. These messages have no place to display
+                // received read receipt anyway.  And since their text
+                // is locally generated, quoting them is dangerous as
+                // it may contain contact names. E.g., for original
+                // message "Group left by me", a read receipt will
+                // quote "Group left by <name>", and the name can be a
+                // display name stored in address book rather than
+                // the name sent in the From field by the user.
                 if msg.param.get_bool(Param::WantsMdn).unwrap_or_default()
+                    && !msg.is_system_message()
                     && context.get_config_bool(Config::MdnsEnabled).await
                 {
                     if let Err(err) = send_mdn(context, &msg).await {
@@ -703,7 +737,7 @@ async fn set_delivered(context: &Context, msg_id: MsgId) {
         )
         .await
         .unwrap_or_default();
-    context.emit_event(Event::MsgDelivered { chat_id, msg_id });
+    context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
 }
 
 /// Constructs a job for sending a message.
@@ -828,25 +862,6 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
-    if let Some(delete_server_after) = context.get_config_delete_server_after().await {
-        let threshold_timestamp = time() - delete_server_after;
-
-        context
-            .sql
-            .query_row_optional(
-                "SELECT id FROM msgs \
-             WHERE timestamp < ? \
-             AND server_uid != 0",
-                paramsv![threshold_timestamp],
-                |row| row.get::<_, MsgId>(0),
-            )
-            .await
-    } else {
-        Ok(None)
-    }
-}
-
 async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         Some(Job::new(
@@ -970,7 +985,6 @@ async fn perform_job_action(
             location::job_maybe_send_locations_ended(context, job).await
         }
         Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
-        Action::OldDeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
@@ -980,10 +994,7 @@ async fn perform_job_action(
         }
     };
 
-    info!(
-        context,
-        "Inbox finished immediate try {} of job {}", tries, job
-    );
+    info!(context, "Finished immediate try {} of job {}", tries, job);
 
     try_res
 }
@@ -1031,7 +1042,6 @@ pub async fn add(context: &Context, job: Job) {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
             | Action::EmptyServer
-            | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
             | Action::MarkseenMsgOnImap
             | Action::MoveMsg => {

src/key.rs

@@ -247,7 +247,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
         Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
+            let start = std::time::SystemTime::now();
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
@@ -258,7 +258,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
+                start.elapsed().unwrap_or_default().as_secs()
             );
             Ok(keypair)
         }

src/lib.rs

@@ -1,6 +1,11 @@
 #![forbid(unsafe_code)]
-#![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![allow(clippy::match_bool)]
+#![deny(
+    clippy::correctness,
+    missing_debug_implementations,
+    clippy::all,
+    clippy::indexing_slicing
+)]
+#![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
 #[macro_use]
 extern crate num_derive;
@@ -43,6 +48,7 @@ pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
+pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;
@@ -73,6 +79,8 @@ mod dehtml;
 pub mod dc_receive_imf;
 pub mod dc_tools;
 
+pub mod accounts;
+
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed
 pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 

src/location.rs

@@ -9,7 +9,7 @@ use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -227,7 +227,7 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
                     .await;
                 chat::add_info_msg(context, chat_id, stock_str).await;
             }
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
             if 0 != seconds {
                 schedule_maybe_send_locations(context, false).await;
                 job::add(
@@ -301,7 +301,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             }
         }
         if continue_streaming {
-            context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
         schedule_maybe_send_locations(context, false).await;
     }
@@ -381,7 +381,7 @@ pub async fn delete_all(context: &Context) -> Result<(), Error> {
         .sql
         .execute("DELETE FROM locations;", paramsv![])
         .await?;
-    context.emit_event(Event::LocationChanged(None));
+    context.emit_event(EventType::LocationChanged(None));
     Ok(())
 }
 
@@ -715,7 +715,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
                 .await;
             chat::add_info_msg(context, chat_id, stock_str).await;
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
         }
     }
     job::Status::Finished(Ok(()))

src/log.rs

@@ -11,7 +11,7 @@ macro_rules! info {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Info(full));
+        emit_event!($ctx, $crate::EventType::Info(full));
     }};
 }
 
@@ -26,7 +26,7 @@ macro_rules! warn {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Warning(full));
+        emit_event!($ctx, $crate::EventType::Warning(full));
     }};
 }
 
@@ -37,7 +37,18 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::Event::Error(formatted));
+        emit_event!($ctx, $crate::EventType::Error(formatted));
+    }};
+}
+
+#[macro_export]
+macro_rules! error_network {
+    ($ctx:expr, $msg:expr) => {
+        error_network!($ctx, $msg,)
+    };
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+        let formatted = format!($msg, $($args),*);
+        emit_event!($ctx, $crate::EventType::ErrorNetwork(formatted));
     }};
 }
 

src/login_param.rs

@@ -5,7 +5,7 @@ use std::fmt;
 
 use crate::context::Context;
 
-#[derive(Copy, Clone, Debug, Display, FromPrimitive)]
+#[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(i32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
@@ -25,7 +25,7 @@ impl Default for CertificateChecks {
     }
 }
 
-#[derive(Default, Debug)]
+#[derive(Default, Debug, Clone)]
 pub struct LoginParam {
     pub addr: String,
     pub mail_server: String,

src/lot.rs

@@ -91,6 +91,9 @@ pub enum LotState {
     /// text1=domain
     QrAccount = 250,
 
+    /// text1=domain, text2=instance pattern
+    QrWebrtcInstance = 260,
+
     /// id=contact
     QrAddr = 320,
 

src/message.rs

@@ -6,12 +6,13 @@ use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Action};
 use crate::lot::{Lot, LotState, Meaning};
 use crate::mimeparser::{FailureReport, SystemMessage};
@@ -68,18 +69,38 @@ impl MsgId {
         self.0 == 0
     }
 
-    /// Whether the message ID is the special marker1 marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_marker1(self) -> bool {
-        self.0 == DC_MSG_ID_MARKER1
+    /// Returns message state.
+    pub async fn get_state(self, context: &Context) -> crate::sql::Result<MessageState> {
+        let result = context
+            .sql
+            .query_get_value_result("SELECT state FROM msgs WHERE id=?", paramsv![self])
+            .await?
+            .unwrap_or_default();
+        Ok(result)
     }
 
-    /// Whether the message ID is the special day marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_daymarker(self) -> bool {
-        self.0 == DC_MSG_ID_DAYMARKER
+    /// Returns true if the message needs to be moved from `folder`.
+    pub async fn needs_move(self, context: &Context, folder: &str) -> Result<bool, Error> {
+        if !context.get_config_bool(Config::MvboxMove).await {
+            return Ok(false);
+        }
+
+        if context.is_mvbox(folder).await {
+            return Ok(false);
+        }
+
+        let msg = Message::load_from_db(context, self).await?;
+
+        if msg.is_setupmessage() {
+            // do not move setup messages;
+            // there may be a non-delta device that wants to handle it
+            return Ok(false);
+        }
+
+        match msg.is_dc_message {
+            MessengerMessage::No => Ok(false),
+            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
+        }
     }
 
     /// Put message into trash chat and delete message text.
@@ -143,16 +164,7 @@ impl MsgId {
 
 impl std::fmt::Display for MsgId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // Would be nice if we could use match here, but no computed values in ranges.
-        if self.0 == DC_MSG_ID_MARKER1 {
-            write!(f, "Msg#Marker1")
-        } else if self.0 == DC_MSG_ID_DAYMARKER {
-            write!(f, "Msg#DayMarker")
-        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
-            write!(f, "Msg#UnknownSpecial")
-        } else {
-            write!(f, "Msg#{}", self.0)
-        }
+        write!(f, "Msg#{}", self.0)
     }
 }
 
@@ -246,6 +258,8 @@ pub struct Message {
     pub(crate) timestamp_sort: i64,
     pub(crate) timestamp_sent: i64,
     pub(crate) timestamp_rcvd: i64,
+    pub(crate) ephemeral_timer: u32,
+    pub(crate) ephemeral_timestamp: i64,
     pub(crate) text: Option<String>,
     pub(crate) rfc724_mid: String,
     pub(crate) in_reply_to: Option<String>,
@@ -288,6 +302,8 @@ impl Message {
                     "    m.timestamp AS timestamp,",
                     "    m.timestamp_sent AS timestamp_sent,",
                     "    m.timestamp_rcvd AS timestamp_rcvd,",
+                    "    m.ephemeral_timer AS ephemeral_timer,",
+                    "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
                     "    m.error AS error,",
@@ -316,6 +332,8 @@ impl Message {
                     msg.timestamp_sort = row.get("timestamp")?;
                     msg.timestamp_sent = row.get("timestamp_sent")?;
                     msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                    msg.ephemeral_timer = row.get("ephemeral_timer")?;
+                    msg.ephemeral_timestamp = row.get("ephemeral_timestamp")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
                     msg.error = row.get("error")?;
@@ -510,6 +528,14 @@ impl Message {
         self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
+    pub fn get_ephemeral_timer(&self) -> u32 {
+        self.ephemeral_timer
+    }
+
+    pub fn get_ephemeral_timestamp(&self) -> i64 {
+        self.ephemeral_timestamp
+    }
+
     pub async fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
         let mut ret = Lot::new();
 
@@ -574,6 +600,11 @@ impl Message {
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
+    pub fn is_system_message(&self) -> bool {
+        let cmd = self.param.get_cmd();
+        cmd != SystemMessage::Unknown
+    }
+
     /// Whether the message is still being created.
     ///
     /// Messages with attachments might be created before the
@@ -612,6 +643,76 @@ impl Message {
         None
     }
 
+    // add room to a webrtc_instance as defined by the corresponding config-value;
+    // the result may still be prefixed by the type
+    pub fn create_webrtc_instance(instance: &str, room: &str) -> String {
+        let (videochat_type, mut url) = Message::parse_webrtc_instance(instance);
+
+        // make sure, there is a scheme in the url
+        if !url.contains(':') {
+            url = format!("https://{}", url);
+        }
+
+        // add/replace room
+        let url = if url.contains("$ROOM") {
+            url.replace("$ROOM", &room)
+        } else {
+            // if there nothing that would separate the room, add a slash as a separator;
+            // this way, urls can be given as "https://meet.jit.si" as well as "https://meet.jit.si/"
+            let maybe_slash = if url.ends_with('/')
+                || url.ends_with('?')
+                || url.ends_with('#')
+                || url.ends_with('=')
+            {
+                ""
+            } else {
+                "/"
+            };
+            format!("{}{}{}", url, maybe_slash, room)
+        };
+
+        // re-add and normalize type
+        match videochat_type {
+            VideochatType::BasicWebrtc => format!("basicwebrtc:{}", url),
+            VideochatType::Jitsi => format!("jitsi:{}", url),
+            VideochatType::Unknown => url,
+        }
+    }
+
+    /// split a webrtc_instance as defined by the corresponding config-value into a type and a url
+    pub fn parse_webrtc_instance(instance: &str) -> (VideochatType, String) {
+        let instance: String = instance.split_whitespace().collect();
+        let mut split = instance.splitn(2, ':');
+        let type_str = split.next().unwrap_or_default().to_lowercase();
+        let url = split.next();
+        match type_str.as_str() {
+            "basicwebrtc" => (
+                VideochatType::BasicWebrtc,
+                url.unwrap_or_default().to_string(),
+            ),
+            "jitsi" => (VideochatType::Jitsi, url.unwrap_or_default().to_string()),
+            _ => (VideochatType::Unknown, instance.to_string()),
+        }
+    }
+
+    pub fn get_videochat_url(&self) -> Option<String> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).1);
+            }
+        }
+        None
+    }
+
+    pub fn get_videochat_type(&self) -> Option<VideochatType> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).0);
+            }
+        }
+        None
+    }
+
     pub fn set_text(&mut self, text: Option<String>) {
         self.text = text;
     }
@@ -891,6 +992,17 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         ret += "\n";
     }
 
+    if msg.ephemeral_timer != 0 {
+        ret += &format!("Ephemeral timer: {}\n", msg.ephemeral_timer);
+    }
+
+    if msg.ephemeral_timestamp != 0 {
+        ret += &format!(
+            "Expires: {}\n",
+            dc_timestamp_to_str(msg.ephemeral_timestamp)
+        );
+    }
+
     if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
@@ -984,18 +1096,30 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     let extension: &str = &path.extension()?.to_str()?.to_lowercase();
     let info = match extension {
-        "mp3" => (Viewtype::Audio, "audio/mpeg"),
+        "3gp" => (Viewtype::Video, "video/3gpp"),
         "aac" => (Viewtype::Audio, "audio/aac"),
-        "mp4" => (Viewtype::Video, "video/mp4"),
-        "webm" => (Viewtype::Video, "video/webm"),
-        "jpg" => (Viewtype::Image, "image/jpeg"),
+        "avi" => (Viewtype::Video, "video/x-msvideo"),
+        "flac" => (Viewtype::Audio, "audio/flac"),
+        "gif" => (Viewtype::Gif, "image/gif"),
         "jpeg" => (Viewtype::Image, "image/jpeg"),
         "jpe" => (Viewtype::Image, "image/jpeg"),
+        "jpg" => (Viewtype::Image, "image/jpeg"),
+        "mov" => (Viewtype::Video, "video/quicktime"),
+        "mp3" => (Viewtype::Audio, "audio/mpeg"),
+        "mp4" => (Viewtype::Video, "video/mp4"),
+        "oga" => (Viewtype::Audio, "audio/ogg"),
+        "ogg" => (Viewtype::Audio, "audio/ogg"),
+        "ogv" => (Viewtype::Video, "video/ogg"),
+        "opus" => (Viewtype::Audio, "audio/ogg"),
         "png" => (Viewtype::Image, "image/png"),
-        "webp" => (Viewtype::Image, "image/webp"),
-        "gif" => (Viewtype::Gif, "image/gif"),
-        "vcf" => (Viewtype::File, "text/vcard"),
+        "spx" => (Viewtype::Audio, "audio/ogg"), // Ogg Speex Profile
+        "svg" => (Viewtype::Image, "image/svg+xml"),
+        "tgs" => (Viewtype::Sticker, "application/x-tgsticker"),
         "vcard" => (Viewtype::File, "text/vcard"),
+        "vcf" => (Viewtype::File, "text/vcard"),
+        "webm" => (Viewtype::Video, "video/webm"),
+        "webp" => (Viewtype::Image, "image/webp"),
+        "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
         _ => {
             return None;
         }
@@ -1032,7 +1156,7 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
     }
 
     if !msg_ids.is_empty() {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -1096,6 +1220,14 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
     let mut send_event = false;
 
     for (id, curr_state, curr_blocked) in msgs.into_iter() {
+        if let Err(err) = id.start_ephemeral_timer(context).await {
+            error!(
+                context,
+                "Failed to start ephemeral timer for message {}: {}", id, err
+            );
+            continue;
+        }
+
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, id, MessageState::InSeen).await;
@@ -1115,7 +1247,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
     }
 
     if send_event {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -1152,7 +1284,7 @@ pub async fn star_msgs(context: &Context, msg_ids: Vec<MsgId>, star: bool) -> bo
         .is_ok()
 }
 
-/// Returns a summary test.
+/// Returns a summary text.
 pub async fn get_summarytext_by_raw(
     viewtype: Viewtype,
     text: Option<impl AsRef<str>>,
@@ -1196,6 +1328,13 @@ pub async fn get_summarytext_by_raw(
                 format!("{} – {}", label, file_name)
             }
         }
+        Viewtype::VideochatInvitation => {
+            append_text = false;
+            context
+                .stock_str(StockMessage::VideochatInvitation)
+                .await
+                .into_owned()
+        }
         _ => {
             if param.get_cmd() != SystemMessage::LocationOnly {
                 "".to_string()
@@ -1275,7 +1414,7 @@ pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl
             )
             .await
         {
-            Ok(_) => context.emit_event(Event::MsgFailed {
+            Ok(_) => context.emit_event(EventType::MsgFailed {
                 chat_id: msg.chat_id,
                 msg_id,
             }),
@@ -1293,7 +1432,7 @@ pub async fn handle_mdn(
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Option<(ChatId, MsgId)> {
-    if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
+    if from_id <= DC_CONTACT_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
         return None;
     }
 
@@ -1450,7 +1589,7 @@ pub(crate) async fn handle_ndn(
                             .await,
                     )
                     .await;
-                    context.emit_event(Event::ChatModified(chat_id));
+                    context.emit_event(EventType::ChatModified(chat_id));
                 }
             }
         }
@@ -1631,6 +1770,7 @@ pub async fn dc_empty_server(context: &Context, flags: u32) {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chat::ChatItem;
     use crate::test_utils as test;
 
     #[test]
@@ -1754,4 +1894,91 @@ mod tests {
             "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
         );
     }
+
+    #[async_std::test]
+    async fn test_parse_webrtc_instance() {
+        let (webrtc_type, url) = Message::parse_webrtc_instance("basicwebrtc:https://foo/bar");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "https://foo/bar");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("bAsIcwEbrTc:url");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "url");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("https://foo/bar?key=val#key=val");
+        assert_eq!(webrtc_type, VideochatType::Unknown);
+        assert_eq!(url, "https://foo/bar?key=val#key=val");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("jitsi:https://j.si/foo");
+        assert_eq!(webrtc_type, VideochatType::Jitsi);
+        assert_eq!(url, "https://j.si/foo");
+    }
+
+    #[async_std::test]
+    async fn test_create_webrtc_instance() {
+        // webrtc_instance may come from an input field of the ui, be pretty tolerant on input
+        let instance = Message::create_webrtc_instance("https://meet.jit.si/", "123");
+        assert_eq!(instance, "https://meet.jit.si/123");
+
+        let instance = Message::create_webrtc_instance("https://meet.jit.si", "456");
+        assert_eq!(instance, "https://meet.jit.si/456");
+
+        let instance = Message::create_webrtc_instance("meet.jit.si", "789");
+        assert_eq!(instance, "https://meet.jit.si/789");
+
+        let instance = Message::create_webrtc_instance("bla.foo?", "123");
+        assert_eq!(instance, "https://bla.foo?123");
+
+        let instance = Message::create_webrtc_instance("jitsi:bla.foo#", "456");
+        assert_eq!(instance, "jitsi:https://bla.foo#456");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room=", "789");
+        assert_eq!(instance, "https://bla.foo#room=789");
+
+        let instance = Message::create_webrtc_instance("https://bla.foo#room", "123");
+        assert_eq!(instance, "https://bla.foo#room/123");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room$ROOM", "123");
+        assert_eq!(instance, "https://bla.foo#room123");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room=$ROOM&after=cont", "234");
+        assert_eq!(instance, "https://bla.foo#room=234&after=cont");
+
+        let instance = Message::create_webrtc_instance("  meet.jit .si ", "789");
+        assert_eq!(instance, "https://meet.jit.si/789");
+
+        let instance = Message::create_webrtc_instance(" basicwebrtc: basic . stuff\n ", "12345ab");
+        assert_eq!(instance, "basicwebrtc:https://basic.stuff/12345ab");
+    }
+
+    #[async_std::test]
+    async fn test_get_width_height() {
+        let t = test::TestContext::new().await;
+
+        // test that get_width() and get_height() are returning some dimensions for images;
+        // (as the device-chat contains a welcome-images, we check that)
+        t.ctx.update_device_chats().await.ok();
+        let (device_chat_id, _) =
+            chat::create_or_lookup_by_contact_id(&t.ctx, DC_CONTACT_ID_DEVICE, Blocked::Not)
+                .await
+                .unwrap();
+
+        let mut has_image = false;
+        let chatitems = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        for chatitem in chatitems {
+            if let ChatItem::Message { msg_id } = chatitem {
+                if let Ok(msg) = Message::load_from_db(&t.ctx, msg_id.clone()).await {
+                    if msg.get_viewtype() == Viewtype::Image {
+                        has_image = true;
+                        // just check that width/height are inside some reasonable ranges
+                        assert!(msg.get_width() > 100);
+                        assert!(msg.get_height() > 100);
+                        assert!(msg.get_width() < 4000);
+                        assert!(msg.get_height() < 4000);
+                    }
+                }
+            }
+        }
+        assert!(has_image);
+    }
 }

src/mimefactory.rs

@@ -9,6 +9,7 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
 use crate::location;
 use crate::message::{self, Message};
@@ -112,12 +113,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 )
                 .await?;
 
-            let command = msg.param.get_cmd();
-
-            if command != SystemMessage::AutocryptSetupMessage
-                && command != SystemMessage::SecurejoinMessage
-                && context.get_config_bool(Config::MdnsEnabled).await
-            {
+            if !msg.is_system_message() && context.get_config_bool(Config::MdnsEnabled).await {
                 req_mdn = true;
             }
         }
@@ -225,7 +221,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .filter(|(_, addr)| addr != &self_addr)
         {
             res.push((
-                Peerstate::from_addr(self.context, addr).await,
+                Peerstate::from_addr(self.context, addr).await?,
                 addr.as_str(),
             ));
         }
@@ -457,21 +453,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         unprotected_headers.push(Header::new("Date".into(), date));
 
-        let os_name = &self.context.os_name;
-        let os_part = os_name
-            .as_ref()
-            .map(|s| format!("/{}", s))
-            .unwrap_or_default();
-        let version = get_version_str();
-
-        // Add a X-Mailer header.
-        // This is only informational for debugging and may be removed in the release.
-        // We do not rely on this header as it may be removed by MTAs.
-
-        unprotected_headers.push(Header::new(
-            "X-Mailer".into(),
-            format!("Delta Chat Core {}{}", version, os_part),
-        ));
         unprotected_headers.push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
 
         if let Loaded::MDN { .. } = self.loaded {
@@ -498,7 +479,16 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(self.context).await?;
 
-        let subject = encode_words(&subject_str);
+        let subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str
+        } else {
+            encode_words(&subject_str)
+        };
 
         let mut message = match self.loaded {
             Loaded::Message { .. } => {
@@ -526,6 +516,14 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
         };
 
+        let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(self.context).await?;
+        if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
+            protected_headers.push(Header::new(
+                "Ephemeral-Timer".to_string(),
+                duration.to_string(),
+            ));
+        }
+
         // we could also store the message-id in the protected headers
         // which would probably help to survive providers like
         // Outlook.com or hotmail which mangle the Message-ID.
@@ -776,6 +774,26 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     "location-streaming-enabled".into(),
                 ));
             }
+            SystemMessage::EphemeralTimerChanged => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "ephemeral-timer-changed".to_string(),
+                ));
+            }
+            SystemMessage::LocationOnly => {
+                // This should prevent automatic replies,
+                // such as non-delivery reports.
+                //
+                // See https://tools.ietf.org/html/rfc3834
+                //
+                // Adding this header without encryption leaks some
+                // information about the message contents, but it can
+                // already be easily guessed from message timing and size.
+                unprotected_headers.push(Header::new(
+                    "Auto-Submitted".to_string(),
+                    "auto-generated".to_string(),
+                ));
+            }
             SystemMessage::AutocryptSetupMessage => {
                 unprotected_headers
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
@@ -837,6 +855,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if self.msg.viewtype == Viewtype::Sticker {
             protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
+        } else if self.msg.viewtype == Viewtype::VideochatInvitation {
+            protected_headers.push(Header::new(
+                "Chat-Content".into(),
+                "videochat-invitation".into(),
+            ));
+            protected_headers.push(Header::new(
+                "Chat-Webrtc-Room".into(),
+                self.msg
+                    .param
+                    .get(Param::WebrtcRoom)
+                    .unwrap_or_default()
+                    .into(),
+            ));
         }
 
         if self.msg.viewtype == Viewtype::Voice
@@ -1379,6 +1410,54 @@ mod tests {
                 .as_bytes(),
         )
         .await;
+
+        // 5. Receive an mdn (read receipt) and make sure the mdn's subject is not used
+        let t = TestContext::new_alice().await;
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@example.com\n\
+            To: Charlie <charlie@example.com>\n\
+            Subject: Hello, Charlie\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        let new_msg = incoming_msg_to_reply_msg(b"From: charlie@example.com\n\
+                 To: alice@example.com\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;charlie@example.com\n\
+                 Final-Recipient: rfc822;charlie@example.com\n\
+                 Original-Message-ID: <2893@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n", &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        // The subject string should not be "Re: message opened"
+        assert_eq!("Re: Hello, Charlie", mf.subject_str().await);
     }
 
     async fn first_subject_str(t: TestContext) -> String {
@@ -1415,7 +1494,7 @@ mod tests {
         mf.subject_str().await
     }
 
-    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    // Creates a `Message` that replies "Hi" to the incoming email in `imf_raw`.
     async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
         context
             .set_config(Config::ShowEmails, Some("2"))

src/mimeparser.rs

@@ -16,14 +16,13 @@ use crate::dc_tools::*;
 use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::error::{bail, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
 use crate::param::*;
 use crate::peerstate::Peerstate;
-use crate::securejoin::handle_degrade_event;
 use crate::simplify::*;
 use crate::stock::StockMessage;
 
@@ -46,7 +45,14 @@ pub struct MimeMessage {
     pub from: Vec<SingleInfo>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
+
+    /// Set of valid signature fingerprints if a message is an
+    /// Autocrypt encrypted and signed message.
+    ///
+    /// If a message is not encrypted or the signature is not valid,
+    /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
+
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
@@ -76,6 +82,9 @@ pub enum SystemMessage {
     SecurejoinMessage = 7,
     LocationStreamingEnabled = 8,
     LocationOnly = 9,
+
+    /// Chat ephemeral message timer is changed.
+    EphemeralTimerChanged = 10,
 }
 
 impl Default for SystemMessage {
@@ -118,53 +127,74 @@ impl MimeMessage {
         let mail_raw;
         let mut gossipped_addr = Default::default();
 
-        let (mail, signatures) = match e2ee::try_decrypt(context, &mail, message_time).await {
-            Ok((raw, signatures)) => {
-                if let Some(raw) = raw {
-                    // Valid autocrypt message, encrypted
-                    mail_raw = raw;
-                    let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
-                    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
-                        info!(context, "decrypted message mime-body:");
-                        println!("{}", String::from_utf8_lossy(&mail_raw));
-                    }
+        let (mail, signatures, warn_empty_signature) =
+            match e2ee::try_decrypt(context, &mail, message_time).await {
+                Ok((raw, signatures)) => {
+                    if let Some(raw) = raw {
+                        // Encrypted, but maybe unsigned message. Only if
+                        // `signatures` set is non-empty, it is a valid
+                        // autocrypt message.
 
-                    // Handle any gossip headers if the mail was encrypted.  See section
-                    // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
-                    let gossip_headers = decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
-                    gossipped_addr =
-                        update_gossip_peerstates(context, message_time, &mail, gossip_headers)
+                        mail_raw = raw;
+                        let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
+                        if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+                            info!(context, "decrypted message mime-body:");
+                            println!("{}", String::from_utf8_lossy(&mail_raw));
+                        }
+
+                        // Handle any gossip headers if the mail was encrypted.  See section
+                        // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
+                        // but only if the mail was correctly signed:
+                        if !signatures.is_empty() {
+                            let gossip_headers =
+                                decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
+                            gossipped_addr = update_gossip_peerstates(
+                                context,
+                                message_time,
+                                &mail,
+                                gossip_headers,
+                            )
                             .await?;
+                        }
 
-                    // let known protected headers from the decrypted
-                    // part override the unencrypted top-level
-                    MimeMessage::merge_headers(
-                        context,
-                        &mut headers,
-                        &mut recipients,
-                        &mut from,
-                        &mut chat_disposition_notification_to,
-                        &decrypted_mail.headers,
-                    );
+                        // let known protected headers from the decrypted
+                        // part override the unencrypted top-level
 
-                    (decrypted_mail, signatures)
-                } else {
-                    // Message was not encrypted
-                    (mail, signatures)
+                        // Signature was checked for original From, so we
+                        // do not allow overriding it.
+                        let mut throwaway_from = from.clone();
+
+                        // We do not want to allow unencrypted subject in encrypted emails because the user might falsely think that the subject is safe.
+                        // See https://github.com/deltachat/deltachat-core-rust/issues/1790.
+                        headers.remove("subject");
+
+                        MimeMessage::merge_headers(
+                            context,
+                            &mut headers,
+                            &mut recipients,
+                            &mut throwaway_from,
+                            &mut chat_disposition_notification_to,
+                            &decrypted_mail.headers,
+                        );
+
+                        (decrypted_mail, signatures, true)
+                    } else {
+                        // Message was not encrypted
+                        (mail, signatures, false)
+                    }
                 }
-            }
-            Err(err) => {
-                // continue with the current, still encrypted, mime tree.
-                // unencrypted parts will be replaced by an error message
-                // that is added as "the message" to the chat then.
-                //
-                // if we just return here, the header is missing
-                // and the caller cannot display the message
-                // and try to assign the message to a chat
-                warn!(context, "decryption failed: {}", err);
-                (mail, Default::default())
-            }
-        };
+                Err(err) => {
+                    // continue with the current, still encrypted, mime tree.
+                    // unencrypted parts will be replaced by an error message
+                    // that is added as "the message" to the chat then.
+                    //
+                    // if we just return here, the header is missing
+                    // and the caller cannot display the message
+                    // and try to assign the message to a chat
+                    warn!(context, "decryption failed: {}", err);
+                    (mail, Default::default(), true)
+                }
+            };
 
         let mut parser = MimeMessage {
             parts: Vec::new(),
@@ -190,6 +220,12 @@ impl MimeMessage {
         parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
+        if warn_empty_signature && parser.signatures.is_empty() {
+            for part in parser.parts.iter_mut() {
+                part.error = "No valid signature".to_string();
+            }
+        }
+
         Ok(parser)
     }
 
@@ -214,6 +250,8 @@ impl MimeMessage {
         } else if let Some(value) = self.get(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
+            } else if value == "ephemeral-timer-changed" {
+                self.is_system_message = SystemMessage::EphemeralTimerChanged;
             }
         }
         Ok(())
@@ -230,10 +268,24 @@ impl MimeMessage {
         }
     }
 
+    fn parse_videochat_headers(&mut self) {
+        if let Some(value) = self.get(HeaderDef::ChatContent).cloned() {
+            if value == "videochat-invitation" {
+                let instance = self.get(HeaderDef::ChatWebrtcRoom).cloned();
+                if let Some(part) = self.parts.first_mut() {
+                    part.typ = Viewtype::VideochatInvitation;
+                    part.param
+                        .set(Param::WebrtcRoom, instance.unwrap_or_default());
+                }
+            }
+        }
+    }
+
     /// Squashes mutlipart chat messages with attachment into single-part messages.
     ///
     /// Delta Chat sends attachments, such as images, in two-part messages, with the first message
     /// containing an explanation. If such a message is detected, first part can be safely dropped.
+    #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
             let need_drop = {
@@ -267,22 +319,21 @@ impl MimeMessage {
     fn parse_attachments(&mut self) {
         // Attachment messages should be squashed into a single part
         // before calling this function.
-        if self.parts.len() == 1 {
-            if self.parts[0].typ == Viewtype::Audio
-                && self.get(HeaderDef::ChatVoiceMessage).is_some()
-            {
-                let part_mut = &mut self.parts[0];
-                part_mut.typ = Viewtype::Voice;
+        if self.parts.len() != 1 {
+            return;
+        }
+
+        if let Some(mut part) = self.parts.pop() {
+            if part.typ == Viewtype::Audio && self.get(HeaderDef::ChatVoiceMessage).is_some() {
+                part.typ = Viewtype::Voice;
             }
-            if self.parts[0].typ == Viewtype::Image {
+            if part.typ == Viewtype::Image {
                 if let Some(value) = self.get(HeaderDef::ChatContent) {
                     if value == "sticker" {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.typ = Viewtype::Sticker;
+                        part.typ = Viewtype::Sticker;
                     }
                 }
             }
-            let part = &self.parts[0];
             if part.typ == Viewtype::Audio
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
@@ -290,17 +341,19 @@ impl MimeMessage {
                 if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.param.set_int(Param::Duration, duration_ms);
+                        part.param.set_int(Param::Duration, duration_ms);
                     }
                 }
             }
+
+            self.parts.push(part);
         }
     }
 
     fn parse_headers(&mut self, context: &Context) -> Result<()> {
         self.parse_system_message_headers(context)?;
         self.parse_avatar_headers();
+        self.parse_videochat_headers();
         self.squash_attachment_parts();
 
         if let Some(ref subject) = self.get_subject() {
@@ -316,12 +369,11 @@ impl MimeMessage {
                 }
             }
             if prepend_subject {
-                let subj = if let Some(n) = subject.find('[') {
-                    &subject[0..n]
-                } else {
-                    subject
-                }
-                .trim();
+                let subj = subject
+                    .find('[')
+                    .and_then(|n| subject.get(..n))
+                    .unwrap_or(subject)
+                    .trim();
 
                 if !subj.is_empty() {
                     for part in self.parts.iter_mut() {
@@ -379,8 +431,7 @@ impl MimeMessage {
             Some(AvatarAction::Delete)
         } else {
             let mut i = 0;
-            while i != self.parts.len() {
-                let part = &mut self.parts[i];
+            while let Some(part) = self.parts.get_mut(i) {
                 if let Some(part_filename) = &part.org_filename {
                     if part_filename == &header_value {
                         if let Some(blob) = part.param.get(Param::File) {
@@ -397,6 +448,11 @@ impl MimeMessage {
         }
     }
 
+    /// Returns true if the message was encrypted as defined in
+    /// Autocrypt standard.
+    ///
+    /// This means the message was both encrypted and signed with a
+    /// valid signature.
     pub fn was_encrypted(&self) -> bool {
         !self.signatures.is_empty()
     }
@@ -544,7 +600,7 @@ impl MimeMessage {
                 contains exactly two body parts.  The first body
                 part is the body part over which the digital signature was created [...]
                 The second body part contains the control information necessary to
-                verify the digital signature." We simpliy take the first body part and
+                verify the digital signature." We simply take the first body part and
                 skip the rest.  (see
                 https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
                 for background information why we use encrypted+signed) */
@@ -765,16 +821,11 @@ impl MimeMessage {
     }
 
     pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
-        if self.parts.is_empty() {
-            return;
+        if let Some(part) = self.parts.first_mut() {
+            part.typ = Viewtype::Text;
+            part.msg = format!("[{}]", error_msg.as_ref());
+            self.parts.truncate(1);
         }
-
-        let part = &mut self.parts[0];
-        part.typ = Viewtype::Text;
-        part.msg = format!("[{}]", error_msg.as_ref());
-        self.parts.truncate(1);
-
-        assert_eq!(self.parts.len(), 1);
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
@@ -825,7 +876,11 @@ impl MimeMessage {
         report: &mailparse::ParsedMail<'_>,
     ) -> Result<Option<Report>> {
         // parse as mailheaders
-        let report_body = report.subparts[1].get_body_raw()?;
+        let report_body = if let Some(subpart) = report.subparts.get(1) {
+            subpart.get_body_raw()?
+        } else {
+            bail!("Report does not have second MIME part");
+        };
         let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
         // must be present
@@ -903,6 +958,7 @@ impl MimeMessage {
     /// Some providers like GMX and Yahoo do not send standard NDNs (Non Delivery notifications).
     /// If you improve heuristics here you might also have to change prefetch_should_download() in imap/mod.rs.
     /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
+    #[allow(clippy::indexing_slicing)]
     async fn heuristically_parse_ndn(&mut self, context: &Context) -> Option<()> {
         let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
             let from = from.to_ascii_lowercase();
@@ -953,20 +1009,16 @@ impl MimeMessage {
                 if let Some((chat_id, msg_id)) =
                     message::handle_mdn(context, from_id, original_message_id, sent_timestamp).await
                 {
-                    context.emit_event(Event::MsgRead { chat_id, msg_id });
+                    context.emit_event(EventType::MsgRead { chat_id, msg_id });
                 }
             }
         }
 
         if let Some(failure_report) = &self.failure_report {
-            let error = parts.iter().find(|p| p.typ == Viewtype::Text).map(|p| {
-                let msg = &p.msg;
-                match msg.find("\n--- ") {
-                    Some(footer_start) => &msg[..footer_start],
-                    None => msg,
-                }
-                .trim()
-            });
+            let error = parts
+                .iter()
+                .find(|p| p.typ == Viewtype::Text)
+                .map(|p| p.msg.clone());
             message::handle_ndn(context, failure_report, error).await
         }
     }
@@ -989,7 +1041,7 @@ async fn update_gossip_peerstates(
                 .iter()
                 .any(|info| info.addr == header.addr.to_lowercase())
             {
-                let mut peerstate = Peerstate::from_addr(context, &header.addr).await;
+                let mut peerstate = Peerstate::from_addr(context, &header.addr).await?;
                 if let Some(ref mut peerstate) = peerstate {
                     peerstate.apply_gossip(header, message_time);
                     peerstate.save_to_db(&context.sql, false).await?;
@@ -999,9 +1051,7 @@ async fn update_gossip_peerstates(
                     peerstate = Some(p);
                 }
                 if let Some(peerstate) = peerstate {
-                    if peerstate.degrade_event.is_some() {
-                        handle_degrade_event(context, &peerstate).await?;
-                    }
+                    peerstate.handle_fingerprint_change(context).await?;
                 }
 
                 gossipped_addr.insert(header.addr.clone());
@@ -1031,6 +1081,7 @@ pub(crate) struct FailureReport {
     pub failed_recipient: Option<String>,
 }
 
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
     // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
     let mut msgids = Vec::new();
@@ -1433,6 +1484,28 @@ mod tests {
         assert!(mimeparser.group_avatar.unwrap().is_change());
     }
 
+    #[async_std::test]
+    async fn test_mimeparser_with_videochat() {
+        let t = TestContext::new().await;
+
+        let raw = include_bytes!("../test-data/message/videochat_invitation.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        assert_eq!(mimeparser.parts.len(), 1);
+        assert_eq!(mimeparser.parts[0].typ, Viewtype::VideochatInvitation);
+        assert_eq!(
+            mimeparser.parts[0]
+                .param
+                .get(Param::WebrtcRoom)
+                .unwrap_or_default(),
+            "https://example.org/p2p/?roomname=6HiduoAn4xN"
+        );
+        assert!(mimeparser.parts[0]
+            .msg
+            .contains("https://example.org/p2p/?roomname=6HiduoAn4xN"));
+        assert_eq!(mimeparser.user_avatar, None);
+        assert_eq!(mimeparser.group_avatar, None);
+    }
+
     #[async_std::test]
     async fn test_mimeparser_message_kml() {
         let context = TestContext::new().await;

src/param.rs

@@ -68,6 +68,9 @@ pub enum Param {
     /// For Messages
     AttachGroupImage = b'A',
 
+    /// For Messages
+    WebrtcRoom = b'V',
+
     /// For Messages: space-separated list of messaged IDs of forwarded copies.
     ///
     /// This is used when a [crate::message::Message] is in the
@@ -171,7 +174,7 @@ impl str::FromStr for Params {
             let key = key.unwrap_or_default().trim();
             let value = value.unwrap_or_default().trim();
 
-            if let Some(key) = Param::from_u8(key.as_bytes()[0]) {
+            if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
                 inner.insert(key, value.to_string());
             } else {
                 bail!("Unknown key: {}", key);

src/peerstate.rs

@@ -5,9 +5,14 @@ use std::fmt;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
+use crate::chat;
+use crate::constants::Blocked;
 use crate::context::Context;
+use crate::error::{bail, Result};
+use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
+use crate::stock::StockMessage;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -38,7 +43,7 @@ pub struct Peerstate<'a> {
     pub verified_key: Option<SignedPublicKey>,
     pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
-    pub degrade_event: Option<DegradeEvent>,
+    pub fingerprint_changed: bool,
 }
 
 impl<'a> PartialEq for Peerstate<'a> {
@@ -55,7 +60,7 @@ impl<'a> PartialEq for Peerstate<'a> {
             && self.verified_key == other.verified_key
             && self.verified_key_fingerprint == other.verified_key_fingerprint
             && self.to_save == other.to_save
-            && self.degrade_event == other.degrade_event
+            && self.fingerprint_changed == other.fingerprint_changed
     }
 }
 
@@ -76,7 +81,7 @@ impl<'a> fmt::Debug for Peerstate<'a> {
             .field("verified_key", &self.verified_key)
             .field("verified_key_fingerprint", &self.verified_key_fingerprint)
             .field("to_save", &self.to_save)
-            .field("degrade_event", &self.degrade_event)
+            .field("fingerprint_changed", &self.fingerprint_changed)
             .finish()
     }
 }
@@ -88,16 +93,6 @@ pub enum ToSave {
     All = 0x02,
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
-#[repr(u8)]
-pub enum DegradeEvent {
-    /// Recoverable by an incoming encrypted mail.
-    EncryptionPaused = 0x01,
-
-    /// Recoverable by a new verify.
-    FingerprintChanged = 0x02,
-}
-
 impl<'a> Peerstate<'a> {
     pub fn new(context: &'a Context, addr: String) -> Self {
         Peerstate {
@@ -114,7 +109,7 @@ impl<'a> Peerstate<'a> {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: None,
-            degrade_event: None,
+            fingerprint_changed: false,
         }
     }
 
@@ -142,8 +137,12 @@ impl<'a> Peerstate<'a> {
         res
     }
 
-    pub async fn from_addr(context: &'a Context, addr: &str) -> Option<Peerstate<'a>> {
-        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, verified_key, verified_key_fingerprint FROM acpeerstates  WHERE addr=? COLLATE NOCASE;";
+    pub async fn from_addr(context: &'a Context, addr: &str) -> Result<Option<Peerstate<'a>>> {
+        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
+                     gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
+                     verified_key, verified_key_fingerprint \
+                     FROM acpeerstates \
+                     WHERE addr=? COLLATE NOCASE;";
         Self::from_stmt(context, query, paramsv![addr]).await
     }
 
@@ -151,7 +150,7 @@ impl<'a> Peerstate<'a> {
         context: &'a Context,
         _sql: &Sql,
         fingerprint: &Fingerprint,
-    ) -> Option<Peerstate<'a>> {
+    ) -> Result<Option<Peerstate<'a>>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
@@ -167,10 +166,10 @@ impl<'a> Peerstate<'a> {
         context: &'a Context,
         query: &str,
         params: Vec<&dyn crate::ToSql>,
-    ) -> Option<Peerstate<'a>> {
-        context
+    ) -> Result<Option<Peerstate<'a>>> {
+        let peerstate = context
             .sql
-            .query_row(query, params, |row| {
+            .query_row_optional(query, params, |row| {
                 /* all the above queries start with this: SELECT
                 addr, last_seen, last_seen_autocrypt, prefer_encrypted,
                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
@@ -186,15 +185,18 @@ impl<'a> Peerstate<'a> {
                 res.public_key_fingerprint = row
                     .get::<_, Option<String>>(7)?
                     .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
+                    .transpose()
+                    .unwrap_or_default();
                 res.gossip_key_fingerprint = row
                     .get::<_, Option<String>>(8)?
                     .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
+                    .transpose()
+                    .unwrap_or_default();
                 res.verified_key_fingerprint = row
                     .get::<_, Option<String>>(10)?
                     .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
+                    .transpose()
+                    .unwrap_or_default();
                 res.public_key = row
                     .get(4)
                     .ok()
@@ -210,8 +212,8 @@ impl<'a> Peerstate<'a> {
 
                 Ok(res)
             })
-            .await
-            .ok()
+            .await?;
+        Ok(peerstate)
     }
 
     pub fn recalc_fingerprint(&mut self) {
@@ -225,7 +227,7 @@ impl<'a> Peerstate<'a> {
             {
                 self.to_save = Some(ToSave::All);
                 if old_public_fingerprint.is_some() {
-                    self.degrade_event = Some(DegradeEvent::FingerprintChanged);
+                    self.fingerprint_changed = true;
                 }
             }
         }
@@ -239,23 +241,51 @@ impl<'a> Peerstate<'a> {
                 || old_gossip_fingerprint != self.gossip_key_fingerprint
             {
                 self.to_save = Some(ToSave::All);
-                if old_gossip_fingerprint.is_some() {
-                    self.degrade_event = Some(DegradeEvent::FingerprintChanged);
+
+                // Warn about gossip key change only if there is no public key obtained from
+                // Autocrypt header, which overrides gossip key.
+                if old_gossip_fingerprint.is_some() && self.public_key_fingerprint.is_none() {
+                    self.fingerprint_changed = true;
                 }
             }
         }
     }
 
     pub fn degrade_encryption(&mut self, message_time: i64) {
-        if self.prefer_encrypt == EncryptPreference::Mutual {
-            self.degrade_event = Some(DegradeEvent::EncryptionPaused);
-        }
-
         self.prefer_encrypt = EncryptPreference::Reset;
         self.last_seen = message_time;
         self.to_save = Some(ToSave::All);
     }
 
+    /// Adds a warning to the chat corresponding to peerstate if fingerprint has changed.
+    pub(crate) async fn handle_fingerprint_change(&self, context: &Context) -> Result<()> {
+        if self.fingerprint_changed {
+            if let Some(contact_id) = context
+                .sql
+                .query_get_value_result(
+                    "SELECT id FROM contacts WHERE addr=?;",
+                    paramsv![self.addr],
+                )
+                .await?
+            {
+                let (contact_chat_id, _) =
+                    chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Deaddrop)
+                        .await
+                        .unwrap_or_default();
+
+                let msg = context
+                    .stock_string_repl_str(StockMessage::ContactSetupChanged, self.addr.clone())
+                    .await;
+
+                chat::add_info_msg(context, contact_chat_id, msg).await;
+                emit_event!(context, EventType::ChatModified(contact_chat_id));
+            } else {
+                bail!("contact with peerstate.addr {:?} not found", &self.addr);
+            }
+        }
+        Ok(())
+    }
+
     pub fn apply_header(&mut self, header: &Aheader, message_time: i64) {
         if self.addr.to_lowercase() != header.addr.to_lowercase() {
             return;
@@ -269,11 +299,6 @@ impl<'a> Peerstate<'a> {
                 || header.prefer_encrypt == EncryptPreference::NoPreference)
                 && header.prefer_encrypt != self.prefer_encrypt
             {
-                if self.prefer_encrypt == EncryptPreference::Mutual
-                    && header.prefer_encrypt != EncryptPreference::Mutual
-                {
-                    self.degrade_event = Some(DegradeEvent::EncryptionPaused);
-                }
                 self.prefer_encrypt = header.prefer_encrypt;
                 self.to_save = Some(ToSave::All)
             }
@@ -400,21 +425,20 @@ impl<'a> Peerstate<'a> {
     }
 
     pub async fn save_to_db(&self, sql: &Sql, create: bool) -> crate::sql::Result<()> {
-        if create {
-            sql.execute(
-                "INSERT INTO acpeerstates (addr) VALUES(?);",
-                paramsv![self.addr],
-            )
-            .await?;
-        }
-
         if self.to_save == Some(ToSave::All) || create {
             sql.execute(
+                if create {
+                "INSERT INTO acpeerstates (last_seen, last_seen_autocrypt, prefer_encrypted, \
+                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
+                 verified_key, verified_key_fingerprint, addr \
+                ) VALUES(?,?,?,?,?,?,?,?,?,?,?)"
+                } else {
                 "UPDATE acpeerstates \
                  SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
                  public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
                  verified_key=?, verified_key_fingerprint=? \
-                 WHERE addr=?;",
+                 WHERE addr=?"
+                },
                 paramsv![
                     self.last_seen,
                     self.last_seen_autocrypt,
@@ -489,7 +513,7 @@ mod tests {
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -499,7 +523,8 @@ mod tests {
 
         let peerstate_new = Peerstate::from_addr(&ctx.ctx, addr)
             .await
-            .expect("failed to load peerstate from db");
+            .expect("failed to load peerstate from db")
+            .expect("no peerstate found in the database");
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
@@ -507,7 +532,8 @@ mod tests {
         let peerstate_new2 =
             Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint())
                 .await
-                .expect("failed to load peerstate from db");
+                .expect("failed to load peerstate from db")
+                .expect("no peerstate found in the database");
         assert_eq!(peerstate, peerstate_new2);
     }
 
@@ -531,7 +557,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -565,7 +591,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -579,7 +605,36 @@ mod tests {
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
-        assert_eq!(peerstate, peerstate_new);
+        assert_eq!(Some(peerstate), peerstate_new);
+    }
+
+    #[async_std::test]
+    async fn test_peerstate_load_db_defaults() {
+        let ctx = crate::test_utils::TestContext::new().await;
+        let addr = "hello@mail.com";
+
+        // Old code created peerstates with this code and updated
+        // other values later.  If UPDATE failed, other columns had
+        // default values, in particular fingerprints were set to
+        // empty strings instead of NULL. This should not be the case
+        // anymore, but the regression test still checks that defaults
+        // can be loaded without errors.
+        ctx.ctx
+            .sql
+            .execute("INSERT INTO acpeerstates (addr) VALUES(?)", paramsv![addr])
+            .await
+            .expect("Failed to write to the database");
+
+        let peerstate = Peerstate::from_addr(&ctx.ctx, addr)
+            .await
+            .expect("Failed to load peerstate from db")
+            .expect("Loaded peerstate is empty");
+
+        // Check that default values for fingerprints are treated like
+        // NULL.
+        assert_eq!(peerstate.public_key_fingerprint, None);
+        assert_eq!(peerstate.gossip_key_fingerprint, None);
+        assert_eq!(peerstate.verified_key_fingerprint, None);
     }
 
     // TODO: don't copy this from stress.rs

src/pgp.rs

@@ -365,11 +365,13 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
         let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
         let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-        ensure!(!msgs.is_empty(), "No valid messages found");
-
-        match msgs[0].get_content()? {
-            Some(content) => Ok(content),
-            None => bail!("Decrypted message is empty"),
+        if let Some(msg) = msgs.first() {
+            match msg.get_content()? {
+                Some(content) => Ok(content),
+                None => bail!("Decrypted message is empty"),
+            }
+        } else {
+            bail!("No valid messages found")
         }
     })
     .await

src/provider/data.rs

@@ -81,6 +81,21 @@ lazy_static::lazy_static! {
         oauth2_authorizer: None,
     };
 
+    // buzon.uy.md: buzon.uy
+    static ref P_BUZON_UY: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/buzon-uy",
+        server: vec![
+            Server { protocol: IMAP, socket: STARTTLS, hostname: "buzon.uy", port: 143, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "buzon.uy", port: 587, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
+
     // chello.at.md: chello.at
     static ref P_CHELLO_AT: Provider = Provider {
         status: Status::OK,
@@ -142,6 +157,9 @@ lazy_static::lazy_static! {
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/dubby-org",
         server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "dubby.org", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "dubby.org", port: 587, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "dubby.org", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: Some(vec![
             ConfigDefault { key: Config::BccSelf, value: "1" },
@@ -181,6 +199,19 @@ lazy_static::lazy_static! {
         oauth2_authorizer: None,
     };
 
+    // firemail.de.md: firemail.at, firemail.de
+    static ref P_FIREMAIL_DE: Provider = Provider {
+        status: Status::PREPARATION,
+        before_login_hint: "Firemail erlaubt nur bei bezahlten Accounts den vollen Zugriff auf das E-Mail-Protokoll. Wenn Sie nicht für Firemail bezahlen, verwenden Sie bitte einen anderen E-Mail-Anbieter.",
+        after_login_hint: "Leider schränkt Firemail die maximale Gruppengröße ein. Je nach Bezahlmodell sind nur 5 bis 30 Gruppenmitglieder erlaubt.",
+        overview_page: "https://providers.delta.chat/firemail-de",
+        server: vec![
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
     // five.chat.md: five.chat
     static ref P_FIVE_CHAT: Provider = Provider {
         status: Status::OK,
@@ -188,6 +219,9 @@ lazy_static::lazy_static! {
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/five-chat",
         server: vec![
+            Server { protocol: IMAP, socket: STARTTLS, hostname: "five.chat", port: 143, username_pattern: EMAIL },
+            Server { protocol: IMAP, socket: SSL, hostname: "five.chat", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "five.chat", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: Some(vec![
             ConfigDefault { key: Config::BccSelf, value: "1" },
@@ -505,6 +539,21 @@ lazy_static::lazy_static! {
         oauth2_authorizer: None,
     };
 
+    // undernet.uy.md: undernet.uy
+    static ref P_UNDERNET_UY: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/undernet-uy",
+        server: vec![
+            Server { protocol: IMAP, socket: STARTTLS, hostname: "undernet.uy", port: 143, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "undernet.uy", port: 587, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: true,
+        oauth2_authorizer: None,
+    };
+
     // vfemail.md: vfemail.net
     static ref P_VFEMAIL: Provider = Provider {
         status: Status::OK,
@@ -518,6 +567,21 @@ lazy_static::lazy_static! {
         oauth2_authorizer: None,
     };
 
+    // vodafone.de.md: vodafone.de, vodafonemail.de
+    static ref P_VODAFONE_DE: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/vodafone-de",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.vodafonemail.de", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.vodafonemail.de", port: 587, username_pattern: EMAIL },
+        ],
+        config_defaults: None,
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
     // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
     static ref P_WEB_DE: Provider = Provider {
         status: Status::PREPARATION,
@@ -556,6 +620,8 @@ lazy_static::lazy_static! {
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yandex-ru",
         server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "imap.yandex.com", port: 993, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: SSL, hostname: "smtp.yandex.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
         strict_tls: true,
@@ -583,6 +649,7 @@ lazy_static::lazy_static! {
         ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
         ("xfinity.com", &*P_COMCAST),
         ("comcast.net", &*P_COMCAST),
@@ -592,6 +659,8 @@ lazy_static::lazy_static! {
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
+        ("firemail.at", &*P_FIREMAIL_DE),
+        ("firemail.de", &*P_FIREMAIL_DE),
         ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
@@ -683,7 +752,10 @@ lazy_static::lazy_static! {
         ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
         ("ukr.net", &*P_UKR_NET),
+        ("undernet.uy", &*P_UNDERNET_UY),
         ("vfemail.net", &*P_VFEMAIL),
+        ("vodafone.de", &*P_VODAFONE_DE),
+        ("vodafonemail.de", &*P_VODAFONE_DE),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),
         ("flirt.ms", &*P_WEB_DE),

src/qr.rs

@@ -12,11 +12,13 @@ use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
 use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
+use crate::message::Message;
 use crate::param::*;
 use crate::peerstate::*;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
+const DCWEBRTC_SCHEME: &str = "DCWEBRTC:";
 const MAILTO_SCHEME: &str = "mailto:";
 const MATMSG_SCHEME: &str = "MATMSG:";
 const VCARD_SCHEME: &str = "BEGIN:VCARD";
@@ -51,6 +53,8 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
         decode_openpgp(context, qr).await
     } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(context, qr)
+    } else if starts_with_ignore_case(qr, DCWEBRTC_SCHEME) {
+        decode_webrtc_instance(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
         decode_mailto(context, qr).await
     } else if qr.starts_with(SMTP_SCHEME) {
@@ -68,6 +72,7 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
 /// scheme: `OPENPGP4FPR:FINGERPRINT#a=ADDR&n=NAME&i=INVITENUMBER&s=AUTH`
 ///     or: `OPENPGP4FPR:FINGERPRINT#a=ADDR&g=GROUPNAME&x=GROUPID&i=INVITENUMBER&s=AUTH`
+#[allow(clippy::indexing_slicing)]
 async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
@@ -138,7 +143,10 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
-    let peerstate = Peerstate::from_fingerprint(context, &context.sql, &fingerprint).await;
+    let peerstate = match Peerstate::from_fingerprint(context, &context.sql, &fingerprint).await {
+        Ok(peerstate) => peerstate,
+        Err(err) => return format_err!("Can't load peerstate: {}", err).into(),
+    };
 
     if invitenumber.is_none() || auth.is_none() {
         if let Some(peerstate) = peerstate {
@@ -187,13 +195,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 }
 
 /// scheme: `DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3`
+#[allow(clippy::indexing_slicing)]
 fn decode_account(_context: &Context, qr: &str) -> Lot {
     let payload = &qr[DCACCOUNT_SCHEME.len()..];
 
     let mut lot = Lot::new();
 
     if let Ok(url) = url::Url::parse(payload) {
-        if url.scheme() == "https" {
+        if url.scheme() == "http" || url.scheme() == "https" {
             lot.state = LotState::QrAccount;
             lot.text1 = url.host_str().map(|x| x.to_string());
         } else {
@@ -208,6 +217,31 @@ fn decode_account(_context: &Context, qr: &str) -> Lot {
     lot
 }
 
+/// scheme: `DCWEBRTC:https://meet.jit.si/$ROOM`
+#[allow(clippy::indexing_slicing)]
+fn decode_webrtc_instance(_context: &Context, qr: &str) -> Lot {
+    let payload = &qr[DCWEBRTC_SCHEME.len()..];
+
+    let mut lot = Lot::new();
+
+    let (_type, url) = Message::parse_webrtc_instance(payload);
+    if let Ok(url) = url::Url::parse(&url) {
+        if url.scheme() == "http" || url.scheme() == "https" {
+            lot.state = LotState::QrWebrtcInstance;
+            lot.text1 = url.host_str().map(|x| x.to_string());
+            lot.text2 = Some(payload.to_string())
+        } else {
+            lot.state = LotState::QrError;
+            lot.text1 = Some(format!("Bad scheme for webrtc instance: {}", payload));
+        }
+    } else {
+        lot.state = LotState::QrError;
+        lot.text1 = Some(format!("Invalid webrtc instance: {}", payload));
+    }
+
+    lot
+}
+
 #[derive(Debug, Deserialize)]
 struct CreateAccountResponse {
     email: String,
@@ -217,7 +251,8 @@ struct CreateAccountResponse {
 /// take a qr of the type DC_QR_ACCOUNT, parse it's parameters,
 /// download additional information from the contained url and set the parameters.
 /// on success, a configure::configure() should be able to log in to the account
-pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+#[allow(clippy::indexing_slicing)]
+async fn set_account_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
 
     let response: Result<CreateAccountResponse, surf::Error> =
@@ -237,9 +272,24 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error
     Ok(())
 }
 
+pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+    match check_qr(context, &qr).await.state {
+        LotState::QrAccount => set_account_from_qr(context, qr).await,
+        LotState::QrWebrtcInstance => {
+            let val = decode_webrtc_instance(context, qr).text2;
+            context
+                .set_config(Config::WebrtcInstance, val.as_deref())
+                .await?;
+            Ok(())
+        }
+        _ => bail!("qr code does not contain config: {}", qr),
+    }
+}
+
 /// Extract address for the mailto scheme.
 ///
 /// Scheme: `mailto:addr...?subject=...&body=..`
+#[allow(clippy::indexing_slicing)]
 async fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let payload = &qr[MAILTO_SCHEME.len()..];
 
@@ -261,6 +311,7 @@ async fn decode_mailto(context: &Context, qr: &str) -> Lot {
 /// Extract address for the smtp scheme.
 ///
 /// Scheme: `SMTP:addr...:subject...:body...`
+#[allow(clippy::indexing_slicing)]
 async fn decode_smtp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[SMTP_SCHEME.len()..];
 
@@ -283,6 +334,7 @@ async fn decode_smtp(context: &Context, qr: &str) -> Lot {
 /// Scheme: `MATMSG:TO:addr...;SUB:subject...;BODY:body...;`
 ///
 /// There may or may not be linebreaks after the fields.
+#[allow(clippy::indexing_slicing)]
 async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     // Does not work when the text `TO:` is used in subject/body _and_ TO: is not the first field.
     // we ignore this case.
@@ -316,14 +368,15 @@ lazy_static! {
 /// Extract address for the matmsg scheme.
 ///
 /// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
+#[allow(clippy::indexing_slicing)]
 async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
         .captures(qr)
-        .map(|caps| {
-            let last_name = &caps[1];
-            let first_name = &caps[2];
+        .and_then(|caps| {
+            let last_name = caps.get(1)?.as_str().trim();
+            let first_name = caps.get(2)?.as_str().trim();
 
-            format!("{} {}", first_name.trim(), last_name.trim())
+            Some(format!("{} {}", first_name, last_name))
         })
         .unwrap_or_default();
 
@@ -611,12 +664,31 @@ mod tests {
         assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
+    #[async_std::test]
+    async fn test_decode_webrtc_instance() {
+        let ctx = TestContext::new().await;
+
+        let res = check_qr(&ctx.ctx, "DCWEBRTC:basicwebrtc:https://basicurl.com/$ROOM").await;
+        assert_eq!(res.get_state(), LotState::QrWebrtcInstance);
+        assert_eq!(res.get_text1().unwrap(), "basicurl.com");
+        assert_eq!(
+            res.get_text2().unwrap(),
+            "basicwebrtc:https://basicurl.com/$ROOM"
+        );
+
+        // Test it again with mixcased "dcWebRTC:" uri scheme
+        let res = check_qr(&ctx.ctx, "dcWebRTC:https://example.org/").await;
+        assert_eq!(res.get_state(), LotState::QrWebrtcInstance);
+        assert_eq!(res.get_text1().unwrap(), "example.org");
+        assert_eq!(res.get_text2().unwrap(), "https://example.org/");
+    }
+
     #[async_std::test]
     async fn test_decode_account_bad_scheme() {
         let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
-            "DCACCOUNT:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "DCACCOUNT:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
@@ -625,10 +697,40 @@ mod tests {
         // Test it again with lowercased "dcaccount:" uri scheme
         let res = check_qr(
             &ctx.ctx,
-            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "dcaccount:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
     }
+
+    #[async_std::test]
+    async fn test_set_config_from_qr() {
+        let ctx = TestContext::new().await;
+
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "badqr:https://example.org/").await;
+        assert!(!res.is_ok());
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "https://no.qr").await;
+        assert!(!res.is_ok());
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "dcwebrtc:https://example.org/").await;
+        assert!(res.is_ok());
+        assert_eq!(
+            ctx.ctx.get_config(Config::WebrtcInstance).await.unwrap(),
+            "https://example.org/"
+        );
+
+        let res =
+            set_config_from_qr(&ctx.ctx, "DCWEBRTC:basicwebrtc:https://foo.bar/?$ROOM&test").await;
+        assert!(res.is_ok());
+        assert_eq!(
+            ctx.ctx.get_config(Config::WebrtcInstance).await.unwrap(),
+            "basicwebrtc:https://foo.bar/?$ROOM&test"
+        );
+    }
 }

src/scheduler.rs

@@ -1,5 +1,3 @@
-#![warn(clippy::indexing_slicing)]
-
 use async_std::prelude::*;
 use async_std::sync::{channel, Receiver, Sender};
 use async_std::task;
@@ -76,6 +74,13 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 }
                 None => {
                     jobs_loaded = 0;
+
+                    // Expunge folder if needed, e.g. if some jobs have
+                    // deleted messages on the server.
+                    if let Err(err) = connection.maybe_close_folder(&ctx).await {
+                        warn!(ctx, "failed to close folder: {:?}", err);
+                    }
+
                     info = if ctx.get_config_bool(Config::InboxWatch).await {
                         fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
                     } else {
@@ -100,7 +105,7 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
             if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "{}", err);
+                error_network!(ctx, "{}", err);
                 return;
             }
 
@@ -122,7 +127,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
         Some(watch_folder) => {
             // connect and fake idle if unable to connect
             if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "imap connection failed: {}", err);
+                warn!(ctx, "imap connection failed: {}", err);
                 return connection.fake_idle(&ctx, None).await;
             }
 

src/securejoin.rs

@@ -12,7 +12,7 @@ use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
 use crate::error::{bail, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::lot::LotState;
@@ -32,7 +32,7 @@ macro_rules! joiner_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::SecurejoinJoinerProgress {
+        $context.emit_event($crate::events::EventType::SecurejoinJoinerProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -45,7 +45,7 @@ macro_rules! inviter_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::SecurejoinInviterProgress {
+        $context.emit_event($crate::events::EventType::SecurejoinInviterProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -352,9 +352,8 @@ async fn send_handshake_msg(
 }
 
 async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32 {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
-    if contacts.len() == 1 {
-        contacts[0]
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        contact_id
     } else {
         0
     }
@@ -365,11 +364,22 @@ async fn fingerprint_equals_sender(
     fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
 ) -> bool {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+            let peerstate = match Peerstate::from_addr(context, contact.get_addr()).await {
+                Ok(peerstate) => peerstate,
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Failed to sender peerstate for {}: {}",
+                        contact.get_addr(),
+                        err
+                    );
+                    return false;
+                }
+            };
 
-    if contacts.len() == 1 {
-        if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
-            if let Some(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
+            if let Some(peerstate) = peerstate {
                 if peerstate.public_key_fingerprint.is_some()
                     && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
@@ -426,6 +436,7 @@ pub(crate) enum HandshakeMessage {
 /// When handle_securejoin_handshake() is called,
 /// the message is not yet filed in the database;
 /// this is done by receive_imf() later on as needed.
+#[allow(clippy::indexing_slicing)]
 pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
     mime_message: &MimeMessage,
@@ -642,7 +653,7 @@ pub(crate) async fn handle_securejoin_handshake(
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await;
             info!(context, "Auth verified.",);
             secure_connection_established(context, contact_chat_id).await;
-            emit_event!(context, Event::ContactsChanged(Some(contact_id)));
+            emit_event!(context, EventType::ContactsChanged(Some(contact_id)));
             inviter_progress!(context, contact_id, 600);
             if join_vg {
                 // the vg-member-added message is special:
@@ -762,7 +773,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 return Ok(abort_retval);
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined).await;
-            emit_event!(context, Event::ContactsChanged(None));
+            emit_event!(context, EventType::ContactsChanged(None));
             let cg_member_added = mime_message
                 .get(HeaderDef::ChatGroupMemberAdded)
                 .map(|s| s.as_str())
@@ -949,7 +960,7 @@ async fn secure_connection_established(context: &Context, contact_chat_id: ChatI
         .stock_string_repl_str(StockMessage::ContactVerified, addr)
         .await;
     chat::add_info_msg(context, contact_chat_id, msg).await;
-    emit_event!(context, Event::ChatModified(contact_chat_id));
+    emit_event!(context, EventType::ChatModified(contact_chat_id));
 }
 
 async fn could_not_establish_secure_connection(
@@ -976,7 +987,7 @@ async fn could_not_establish_secure_connection(
 
 async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
     if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await
+        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await?
     {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
@@ -1010,65 +1021,18 @@ fn encrypted_and_signed(
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
         false
-    } else if mimeparser.signatures.is_empty() {
-        warn!(context, "Message not signed.",);
-        false
-    } else if expected_fingerprint.is_none() {
+    } else if let Some(expected_fingerprint) = expected_fingerprint {
+        if !mimeparser.signatures.contains(expected_fingerprint) {
+            warn!(
+                context,
+                "Message does not match expected fingerprint {}.", expected_fingerprint,
+            );
+            false
+        } else {
+            true
+        }
+    } else {
         warn!(context, "Fingerprint for comparison missing.");
         false
-    } else if !mimeparser
-        .signatures
-        .contains(expected_fingerprint.unwrap())
-    {
-        warn!(
-            context,
-            "Message does not match expected fingerprint {}.",
-            expected_fingerprint.unwrap(),
-        );
-        false
-    } else {
-        true
     }
 }
-
-pub async fn handle_degrade_event(
-    context: &Context,
-    peerstate: &Peerstate<'_>,
-) -> Result<(), Error> {
-    // - we do not issue an warning for DC_DE_ENCRYPTION_PAUSED as this is quite normal
-    // - currently, we do not issue an extra warning for DC_DE_VERIFICATION_LOST - this always comes
-    //   together with DC_DE_FINGERPRINT_CHANGED which is logged, the idea is not to bother
-    //   with things they cannot fix, so the user is just kicked from the verified group
-    //   (and he will know this and can fix this)
-    if Some(DegradeEvent::FingerprintChanged) == peerstate.degrade_event {
-        let contact_id: i32 = match context
-            .sql
-            .query_get_value(
-                context,
-                "SELECT id FROM contacts WHERE addr=?;",
-                paramsv![peerstate.addr],
-            )
-            .await
-        {
-            None => bail!(
-                "contact with peerstate.addr {:?} not found",
-                &peerstate.addr
-            ),
-            Some(contact_id) => contact_id,
-        };
-        if contact_id > 0 {
-            let (contact_chat_id, _) =
-                chat::create_or_lookup_by_contact_id(context, contact_id as u32, Blocked::Deaddrop)
-                    .await
-                    .unwrap_or_default();
-
-            let msg = context
-                .stock_string_repl_str(StockMessage::ContactSetupChanged, peerstate.addr.clone())
-                .await;
-
-            chat::add_info_msg(context, contact_chat_id, msg).await;
-            emit_event!(context, Event::ChatModified(contact_chat_id));
-        }
-    }
-    Ok(())
-}

src/simplify.rs

@@ -7,14 +7,15 @@
 // but for non-delta-compatibility, that seems to be better.
 // (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
 pub fn escape_message_footer_marks(text: &str) -> String {
-    if text.starts_with("--") {
-        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
+    if let Some(text) = text.strip_prefix("--") {
+        "-\u{200B}-".to_string() + &text.replace("\n--", "\n-\u{200B}-")
     } else {
         text.replace("\n--", "\n-\u{200B}-")
     }
 }
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
+#[allow(clippy::indexing_slicing)]
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
     let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
@@ -41,12 +42,11 @@ fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
 
 /// Remove nonstandard footer and a boolean indicating whether such
 /// footer was removed.
+#[allow(clippy::indexing_slicing)]
 fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     for (ix, &line) in lines.iter().enumerate() {
         if line == "--"
-            || line == "---"
-            || line == "----"
-            || line.starts_with("-----")
+            || line.starts_with("---")
             || line.starts_with("_____")
             || line.starts_with("=====")
             || line.starts_with("*****")
@@ -107,6 +107,7 @@ fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     }
 }
 
+#[allow(clippy::indexing_slicing)]
 fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     let mut last_quoted_line = None;
     for (l, line) in lines.iter().enumerate().rev() {
@@ -132,6 +133,7 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     }
 }
 
+#[allow(clippy::indexing_slicing)]
 fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     let mut last_quoted_line = None;
     let mut has_quoted_headline = false;
@@ -332,9 +334,17 @@ mod tests {
         let (plain, _) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
 
+        // Nonstandard footer sent by https://siju.es/
+        let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
+        let (plain, _) = simplify(input.clone(), false);
+        assert_eq!(plain, "Message text here [...]");
+        let (plain, _) = simplify(input.clone(), true);
+        assert_eq!(plain, input);
+
         let input = "--\ntreated as footer when unescaped".to_string();
         let (plain, _) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
+
         let escaped = escape_message_footer_marks(&input);
         let (plain, _) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");

src/smtp/mod.rs

@@ -1,17 +1,15 @@
 //! # SMTP transport module
 
-#![forbid(clippy::indexing_slicing)]
-
 pub mod send;
 
-use std::time::{Duration, Instant};
+use std::time::{Duration, SystemTime};
 
 use async_smtp::smtp::client::net::*;
 use async_smtp::*;
 
 use crate::constants::*;
 use crate::context::Context;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::oauth2::*;
 use crate::provider::get_provider_info;
@@ -57,7 +55,7 @@ pub(crate) struct Smtp {
     /// Timestamp of last successful send/receive network interaction
     /// (eg connect or send succeeded). On initialization and disconnect
     /// it is set to None.
-    last_success: Option<Instant>,
+    last_success: Option<SystemTime>,
 }
 
 impl Smtp {
@@ -78,7 +76,11 @@ impl Smtp {
     /// have been successfully used the last 60 seconds
     pub async fn has_maybe_stale_connection(&self) -> bool {
         if let Some(last_success) = self.last_success {
-            Instant::now().duration_since(last_success).as_secs() > 60
+            SystemTime::now()
+                .duration_since(last_success)
+                .unwrap_or_default()
+                .as_secs()
+                > 60
         } else {
             false
         }
@@ -100,7 +102,7 @@ impl Smtp {
         }
 
         if lp.send_server.is_empty() || lp.send_port == 0 {
-            context.emit_event(Event::ErrorNetwork("SMTP bad parameters.".into()));
+            context.emit_event(EventType::ErrorNetwork("SMTP bad parameters.".into()));
             return Err(Error::BadParameters);
         }
 
@@ -181,18 +183,18 @@ impl Smtp {
                 .stock_string_repl_str2(
                     StockMessage::ServerResponse,
                     format!("SMTP {}:{}", domain, port),
-                    format!("{}, ({:?})", err.to_string(), err),
+                    err.to_string(),
                 )
                 .await;
 
-            emit_event!(context, Event::ErrorNetwork(message));
+            emit_event!(context, EventType::ErrorNetwork(message));
             return Err(Error::ConnectionFailure(err));
         }
 
         self.transport = Some(trans);
-        self.last_success = Some(Instant::now());
+        self.last_success = Some(SystemTime::now());
 
-        context.emit_event(Event::SmtpConnected(format!(
+        context.emit_event(EventType::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
             lp.send_user,
         )));

src/smtp/send.rs

@@ -4,7 +4,8 @@ use super::Smtp;
 use async_smtp::*;
 
 use crate::context::Context;
-use crate::events::Event;
+use crate::events::EventType;
+use std::time::Duration;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -30,7 +31,7 @@ impl Smtp {
         message: Vec<u8>,
         job_id: u32,
     ) -> Result<()> {
-        let message_len = message.len();
+        let message_len_bytes = message.len();
 
         let recipients_display = recipients
             .iter()
@@ -47,13 +48,18 @@ impl Smtp {
         );
 
         if let Some(ref mut transport) = self.transport {
-            transport.send(mail).await.map_err(Error::SendError)?;
+            // The timeout is 1min + 3min per MB.
+            let timeout = 60 + (180 * message_len_bytes / 1_000_000) as u64;
+            transport
+                .send_with_timeout(mail, Some(&Duration::from_secs(timeout)))
+                .await
+                .map_err(Error::SendError)?;
 
-            context.emit_event(Event::SmtpMessageSent(format!(
+            context.emit_event(EventType::SmtpMessageSent(format!(
                 "Message len={} was smtp-sent to {}",
-                message_len, recipients_display
+                message_len_bytes, recipients_display
             )));
-            self.last_success = Some(std::time::Instant::now());
+            self.last_success = Some(std::time::SystemTime::now());
 
             Ok(())
         } else {

src/sql.rs

@@ -13,6 +13,7 @@ use crate::chat::{update_device_icon, update_saved_messages_icon};
 use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::start_ephemeral_timers;
 use crate::param::*;
 use crate::peerstate::*;
 
@@ -568,16 +569,24 @@ pub async fn housekeeping(context: &Context) {
         }
     }
 
+    if let Err(err) = start_ephemeral_timers(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot start ephemeral timers: {}", err
+        );
+    }
+
     if let Err(err) = prune_tombstones(context).await {
         warn!(
             context,
-            "Houskeeping: Cannot prune message tombstones: {}", err
+            "Housekeeping: Cannot prune message tombstones: {}", err
         );
     }
 
     info!(context, "Housekeeping done.",);
 }
 
+#[allow(clippy::indexing_slicing)]
 fn is_file_in_use(files_in_use: &HashSet<String>, namespc_opt: Option<&str>, name: &str) -> bool {
     let name_to_check = if let Some(namespc) = namespc_opt {
         let name_len = name.len();
@@ -593,11 +602,9 @@ fn is_file_in_use(files_in_use: &HashSet<String>, namespc_opt: Option<&str>, nam
 }
 
 fn maybe_add_file(files_in_use: &mut HashSet<String>, file: impl AsRef<str>) {
-    if !file.as_ref().starts_with("$BLOBDIR/") {
-        return;
+    if let Some(file) = file.as_ref().strip_prefix("$BLOBDIR/") {
+        files_in_use.insert(file.to_string());
     }
-
-    files_in_use.insert(file.as_ref()[9..].into());
 }
 
 async fn maybe_add_from_param(
@@ -1250,6 +1257,37 @@ async fn open(
             .await?;
             sql.set_raw_config_int(context, "dbversion", 64).await?;
         }
+        if dbversion < 65 {
+            info!(context, "[migration] v65");
+            sql.execute(
+                "ALTER TABLE chats ADD COLUMN ephemeral_timer INTEGER",
+                paramsv![],
+            )
+            .await?;
+            // Timer value in seconds. For incoming messages this
+            // timer starts when message is read, so we want to have
+            // the value stored here until the timer starts.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            // Timestamp indicating when the message should be
+            // deleted. It is convenient to store it here because UI
+            // needs this value to display how much time is left until
+            // the message is deleted.
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 65).await?;
+        }
+        if dbversion < 66 {
+            info!(context, "[migration] v66");
+            update_icons = true;
+            sql.set_raw_config_int(context, "dbversion", 66).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1270,7 +1308,7 @@ async fn open(
                 )
                 .await?;
             for addr in &addrs {
-                if let Some(ref mut peerstate) = Peerstate::from_addr(context, addr).await {
+                if let Some(ref mut peerstate) = Peerstate::from_addr(context, addr).await? {
                     peerstate.recalc_fingerprint();
                     peerstate.save_to_db(sql, false).await?;
                 }

src/stock.rs

@@ -130,7 +130,9 @@ pub enum StockMessage {
     ))]
     AcSetupMsgBody = 43,
 
-    #[strum(props(fallback = "Cannot login as %1$s."))]
+    #[strum(props(
+        fallback = "Cannot login as \"%1$s\". Please check if the email address and the password are correct."
+    ))]
     CannotLogin = 60,
 
     #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
@@ -177,7 +179,7 @@ pub enum StockMessage {
                     however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
     WelcomeMessage = 71,
 
-    #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
+    #[strum(props(fallback = "Unknown sender for this chat. See 'info' for more details."))]
     UnknownSenderForChat = 72,
 
     #[strum(props(fallback = "Message from %1$s"))]
@@ -185,6 +187,35 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Failed to send message to %1$s."))]
     FailedSendingTo = 74,
+
+    #[strum(props(fallback = "Message deletion timer is disabled."))]
+    MsgEphemeralTimerDisabled = 75,
+
+    // A fallback message for unknown timer values.
+    // "s" stands for "second" SI unit here.
+    #[strum(props(fallback = "Message deletion timer is set to %1$s s."))]
+    MsgEphemeralTimerEnabled = 76,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 minute."))]
+    MsgEphemeralTimerMinute = 77,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 hour."))]
+    MsgEphemeralTimerHour = 78,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 day."))]
+    MsgEphemeralTimerDay = 79,
+
+    #[strum(props(fallback = "Message deletion timer is set to 1 week."))]
+    MsgEphemeralTimerWeek = 80,
+
+    #[strum(props(fallback = "Message deletion timer is set to 4 weeks."))]
+    MsgEphemeralTimerFourWeeks = 81,
+
+    #[strum(props(fallback = "Video chat invitation"))]
+    VideochatInvitation = 82,
+
+    #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
+    VideochatInviteMsgBody = 83,
 }
 
 /*
@@ -334,10 +365,10 @@ impl Context {
         let action1 = action.trim_end_matches('.');
         match from_id {
             0 => action,
-            1 => {
+            DC_CONTACT_ID_SELF => {
                 self.stock_string_repl_str(StockMessage::MsgActionByMe, action1)
                     .await
-            } // DC_CONTACT_ID_SELF
+            }
             _ => {
                 let displayname = Contact::get_by_id(self, from_id)
                     .await

src/test_utils.rs

@@ -27,9 +27,14 @@ impl TestContext {
     ///
     /// [Context]: crate::context::Context
     pub async fn new() -> Self {
+        use rand::Rng;
+
         let dir = tempdir().unwrap();
         let dbfile = dir.path().join("db.sqlite");
-        let ctx = Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        let id = rand::thread_rng().gen();
+        let ctx = Context::new("FakeOS".into(), dbfile.into(), id)
+            .await
+            .unwrap();
         Self { ctx, dir }
     }
 

test-data/message/videochat_invitation.eml

@@ -0,0 +1,15 @@
+Content-Type: text/plain; charset=utf-8
+Subject: Message from user
+Message-ID: <Mr.f1O61111evx.ikocf333353@example.org>
+Date: Mon, 20 Jul 2020 14:28:30 +0000
+X-Mailer: Delta Chat Core 1.40.0/CLI
+Chat-Version: 1.0
+Chat-Content: videochat-invitation
+Chat-Webrtc-Room: https://example.org/p2p/?roomname=6HiduoAn4xN
+To: <tunis3@example.org>
+From: "=?utf-8?q??=" <tunis4@example.org>
+
+You are invited to an videochat, click https://example.org/p2p/?roomname=6HiduoAn4xN to join.
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat

tests/stress.rs

@@ -99,9 +99,14 @@ struct TestContext {
 }
 
 async fn create_test_context() -> TestContext {
+    use rand::Rng;
+
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    let ctx = Context::new("FakeOs".into(), dbfile.into()).await.unwrap();
+    let id = rand::thread_rng().gen();
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), id)
+        .await
+        .unwrap();
     TestContext { ctx, dir }
 }